/**
* get the region with a int ip address with memory binary search algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock memorySearch(long ip)
{
int blen = IndexBlock.getIndexBlockLength();
if (dbBinStr == null)
{
dbBinStr = new byte[(int)raf.length()];
raf.seek(0L);
raf.readFully(dbBinStr, 0, dbBinStr.Length);
//initialize the global vars
firstIndexPtr = Util.getIntLong(dbBinStr, 0);
lastIndexPtr = Util.getIntLong(dbBinStr, 4);
totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1;
}
//search the index blocks to define the data
int l = 0, h = totalIndexBlocks;
long sip, eip, dataptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
int p = (int)(firstIndexPtr + m * blen);
sip = Util.getIntLong(dbBinStr, p);
if (ip < sip)
{
h = m - 1;
}
else
{
eip = Util.getIntLong(dbBinStr, p + 4);
if (ip > eip)
{
l = m + 1;
}
else
{
dataptr = Util.getIntLong(dbBinStr, p + 8);
break;
}
}
}
//not matched
if (dataptr == 0)
{
return(null);
}
//get the data
int dataLen = (int)((dataptr >> 24) & 0xFF);
int dataPtr = (int)((dataptr & 0x00FFFFFF));
int city_id = (int)Util.getIntLong(dbBinStr, dataPtr);
//String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8");
var region = ReadString(dbBinStr, dataPtr + 4, dataLen - 4);
return(new DataBlock(city_id, region, dataPtr));
}
/**
* construct method with self-define std ip2region bianry string support
* Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL
*
* @param dbConfig
* @param dbBinStr
*/
public DbSearcher(DbConfig dbConfig, byte[] dbBinStr)
{
this.dbConfig = dbConfig;
this.dbBinStr = dbBinStr;
firstIndexPtr = Util.getIntLong(dbBinStr, 0);
lastIndexPtr = Util.getIntLong(dbBinStr, 4);
totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / IndexBlock.getIndexBlockLength()) + 1;
}
/**
* internal method to add a new data block record
*
* @param raf
* @param startIp
* @param endIp
* @param region data
*/
private void addDataBlock(
RandomAccessFile raf,
String startIp, String endIp, String region)
{
byte[] data = region.getBytes("UTF-8");
int dataPtr = 0;
/*byte[] city = new byte[4];
* int city_id = getCityId(region);
* Util.writeIntLong(city, 0, city_id);
* dataPtr = (int)raf.getFilePointer();
* raf.write(city);
* raf.write(data);*/
//check region ptr pool first
if (regionPtrPool.containsKey(region))
{
DataBlock dataBlock = regionPtrPool.get(region);
dataPtr = dataBlock.getDataPtr();
Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
}
else
{
byte[] city = new byte[4];
int city_id = getCityId(region);
Util.writeIntLong(city, 0, city_id);
dataPtr = (int)raf.getFilePointer();
raf.write(city);
raf.write(data);
regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr));
}
//add the data index blocks
IndexBlock ib = new IndexBlock(
Util.ip2long(startIp),
Util.ip2long(endIp),
dataPtr,
data.Length + 4 //4 bytes for the city id
);
indexPool.add(ib);
}
/**
* make the Db file
*
* @param dbFile target output file path
* @throws IOException
*/
public void make(String dbFile)
{
//check and load the gloabl region
if (globalRegionFile != null)
{
Console.WriteLine("+-Try to load the global region data ...");
StreamReader greader = new StreamReader((globalRegionFile));
String gline = null;
while (!greader.EndOfStream)
{
gline = greader.ReadLine();// != null
if (String.IsNullOrWhiteSpace(gline))
{
continue;
}
String[] p = gline.Split(",");
if (p.Length != 5)
{
continue;
}
//push the mapping
globalRegionMap.put(p[2], Int32.Parse(p[0]));
}
greader.Close();
Console.WriteLine("|--[Ok]");
}
//alloc the header size
StreamReader reader = new StreamReader(this.ipSrcFile);
RandomAccessFile raf = new RandomAccessFile(dbFile);
//init the db file
initDbFile(raf);
Console.WriteLine("+-Db file initialized.");
//analysis main loop
Console.WriteLine("+-Try to write the data blocks ... ");
String line = null;
while (!reader.EndOfStream)
{
//line = reader.ReadLine()
line = reader.ReadLine();
if (String.IsNullOrWhiteSpace(line))
{
continue;
}
line = line.Trim();
if (line.Length == 0)
{
continue;
}
if (line[0] == '#')
{
continue;
}
//1. get the start ip
int sIdx = 0, eIdx = 0;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1)
{
continue;
}
String startIp = line.Substring(sIdx, eIdx);
//2. get the end ip
sIdx = eIdx + 1;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1)
{
continue;
}
String endIp = line.Substring(sIdx, eIdx - sIdx);
//3. get the region
sIdx = eIdx + 1;
String region = line.Substring(sIdx);
Console.WriteLine("+-Try to process item " + line);
addDataBlock(raf, startIp, endIp, region);
Console.WriteLine("|--[Ok]");
}
Console.WriteLine("|--Data block flushed!");
Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n");
//write the index bytes
Console.WriteLine("+-Try to write index blocks ... ");
//record the start block
IndexBlock indexBlock = null;
HeaderBlock hb = null;
indexBlock = indexPool.First();
long indexStartIp = indexBlock.getStartIp(),
indexStratPtr = raf.getFilePointer(), indexEndPtr;
headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr)));
int blockLength = IndexBlock.getIndexBlockLength();
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;
//Iterator<IndexBlock> indexIt = indexPool.iterator();
//while ( indexIt.hasNext() ) {
foreach (var block in indexPool)
{
if (++counter >= shotCounter)
{
hb = new HeaderBlock(
block.getStartIp(),
(int)raf.getFilePointer()
);
headerPool.add(hb);
counter = 0;
}
//write the buffer
raf.write(block.getBytes());
}
//record the end block
if (counter > 0)
{
indexBlock = indexPool.Last();
hb = new HeaderBlock(
indexBlock.getStartIp(),
((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength()
);
headerPool.add(hb);
}
indexEndPtr = raf.getFilePointer();
Console.WriteLine("|--[Ok]");
//write the super blocks
Console.WriteLine("+-Try to write the super blocks ... ");
raf.seek(0L); //reset the file pointer
byte[] superBuffer = new byte[8];
Util.writeIntLong(superBuffer, 0, indexStratPtr);
Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength);
raf.write(superBuffer);
Console.WriteLine("|--[Ok]");
//write the header blocks
Console.WriteLine("+-Try to write the header blocks ... ");
//Iterator<HeaderBlock> headerIt = headerPool.iterator();
//while ( headerIt.hasNext() ) {
// HeaderBlock headerBlock = headerIt.next();
// raf.write(headerBlock.getBytes());
//}
foreach (var headerBlock in headerPool)
{
raf.write(headerBlock.getBytes());
}
//write the copyright and the release timestamp info
Console.WriteLine("+-Try to write the copyright and release date info ... ");
raf.seek(raf.length());
//Calendar cal = Calendar.getInstance();
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
var date = DateTime.Now;
String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd");
var timespan = new DateTimeOffset(date).ToUnixTimeSeconds();
raf.write((int)(timespan)); //the unix timestamp
raf.write(copyright.getBytes("UTF-8"));
Console.WriteLine("|--[Ok]");
reader.Close();
raf.close();
}
/**
* get the region with a int ip address with binary search algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock binarySearch(long ip)
{
int blen = IndexBlock.getIndexBlockLength();
if (totalIndexBlocks == 0)
{
raf.seek(0L);
byte[] superBytes = new byte[8];
raf.readFully(superBytes, 0, superBytes.Length);
//initialize the global vars
firstIndexPtr = Util.getIntLong(superBytes, 0);
lastIndexPtr = Util.getIntLong(superBytes, 4);
totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1;
}
//search the index blocks to define the data
int l = 0, h = totalIndexBlocks;
byte[] buffer = new byte[blen];
long sip, eip, dataptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
raf.seek(firstIndexPtr + m * blen); //set the file pointer
raf.readFully(buffer, 0, buffer.Length);
sip = Util.getIntLong(buffer, 0);
if (ip < sip)
{
h = m - 1;
}
else
{
eip = Util.getIntLong(buffer, 4);
if (ip > eip)
{
l = m + 1;
}
else
{
dataptr = Util.getIntLong(buffer, 8);
break;
}
}
}
//not matched
if (dataptr == 0)
{
return(null);
}
//get the data
int dataLen = (int)((dataptr >> 24) & 0xFF);
int dataPtr = (int)((dataptr & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.Length);
int city_id = (int)Util.getIntLong(data, 0);
//String region = new String(data, 4, data.Length - 4, "UTF-8");
var region = ReadString(data, 4, data.Length - 4);
return(new DataBlock(city_id, region, dataPtr));
}
/**
* get the region with a int ip address with b-tree algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock btreeSearch(long ip)
{
//check and load the header
if (HeaderSip == null)
{
raf.seek(8L); //pass the super block
byte[] b = new byte[dbConfig.getTotalHeaderSize()];
// byte[] b = new byte[4096];
raf.readFully(b, 0, b.Length);
//fill the header
int len = b.Length >> 3, idx = 0; //b.lenght / 8
HeaderSip = new long[len];
HeaderPtr = new int [len];
long startIp, xDataPtr;
for (int i = 0; i < b.Length; i += 8)
{
startIp = Util.getIntLong(b, i);
xDataPtr = Util.getIntLong(b, i + 4);
if (xDataPtr == 0)
{
break;
}
HeaderSip[idx] = startIp;
HeaderPtr[idx] = (int)xDataPtr;
idx++;
}
headerLength = idx;
}
//1. define the index block with the binary search
if (ip == HeaderSip[0])
{
return(getByIndexPtr(HeaderPtr[0]));
}
else if (ip == HeaderSip[headerLength - 1])
{
return(getByIndexPtr(HeaderPtr[headerLength - 1]));
}
int l = 0, h = headerLength, sptr = 0, eptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
//perfetc matched, just return it
if (ip == HeaderSip[m])
{
if (m > 0)
{
sptr = HeaderPtr[m - 1];
eptr = HeaderPtr[m];
}
else
{
sptr = HeaderPtr[m];
eptr = HeaderPtr[m + 1];
}
break;
}
//less then the middle value
if (ip < HeaderSip[m])
{
if (m == 0)
{
sptr = HeaderPtr[m];
eptr = HeaderPtr[m + 1];
break;
}
else if (ip > HeaderSip[m - 1])
{
sptr = HeaderPtr[m - 1];
eptr = HeaderPtr[m];
break;
}
h = m - 1;
}
else
{
if (m == headerLength - 1)
{
sptr = HeaderPtr[m - 1];
eptr = HeaderPtr[m];
break;
}
else if (ip <= HeaderSip[m + 1])
{
sptr = HeaderPtr[m];
eptr = HeaderPtr[m + 1];
break;
}
l = m + 1;
}
}
//match nothing just stop it
if (sptr == 0)
{
return(null);
}
//2. search the index blocks to define the data
int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength();
byte[] iBuffer = new byte[blockLen + blen]; //include the right border block
raf.seek(sptr);
raf.readFully(iBuffer, 0, iBuffer.Length);
l = 0;
h = blockLen / blen;
long sip, eip, dataptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
int p = m * blen;
sip = Util.getIntLong(iBuffer, p);
if (ip < sip)
{
h = m - 1;
}
else
{
eip = Util.getIntLong(iBuffer, p + 4);
if (ip > eip)
{
l = m + 1;
}
else
{
dataptr = Util.getIntLong(iBuffer, p + 8);
break;
}
}
}
//not matched
if (dataptr == 0)
{
return(null);
}
//3. get the data
int dataLen = (int)((dataptr >> 24) & 0xFF);
int dataPtr = (int)((dataptr & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.Length);
int city_id = (int)Util.getIntLong(data, 0);
//String region = new String(data, 4, data.Length - 4, "UTF-8");
var region = ReadString(data, 4, data.Length - 4);
return(new DataBlock(city_id, region, dataPtr));
}
