/**
* initialize the db file
*
* @param raf
* @throws IOException
*/
private void initDbFile(RandomAccessFile raf)
{
//1. zero fill the header part
raf.seek(0L);
raf.write(new byte[8]); //super block
raf.write(new byte[dbConfig.getTotalHeaderSize()]); //header block
headerPool = new List <HeaderBlock>(); //new LinkedList<HeaderBlock>();
indexPool = new List <IndexBlock>(); //new LinkedList<IndexBlock>();
}
/**
* internal method to add a new data block record
*
* @param raf
* @param startIp
* @param endIp
* @param region data
*/
private void addDataBlock(
RandomAccessFile raf,
String startIp, String endIp, String region)
{
byte[] data = region.getBytes("UTF-8");
int dataPtr = 0;
/*byte[] city = new byte[4];
* int city_id = getCityId(region);
* Util.writeIntLong(city, 0, city_id);
* dataPtr = (int)raf.getFilePointer();
* raf.write(city);
* raf.write(data);*/
//check region ptr pool first
if (regionPtrPool.containsKey(region))
{
DataBlock dataBlock = regionPtrPool.get(region);
dataPtr = dataBlock.getDataPtr();
Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
}
else
{
byte[] city = new byte[4];
int city_id = getCityId(region);
Util.writeIntLong(city, 0, city_id);
dataPtr = (int)raf.getFilePointer();
raf.write(city);
raf.write(data);
regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr));
}
//add the data index blocks
IndexBlock ib = new IndexBlock(
Util.ip2long(startIp),
Util.ip2long(endIp),
dataPtr,
data.Length + 4 //4 bytes for the city id
);
indexPool.add(ib);
}
/**
* make the Db file
*
* @param dbFile target output file path
* @throws IOException
*/
public void make(String dbFile)
{
//check and load the gloabl region
if (globalRegionFile != null)
{
Console.WriteLine("+-Try to load the global region data ...");
StreamReader greader = new StreamReader((globalRegionFile));
String gline = null;
while (!greader.EndOfStream)
{
gline = greader.ReadLine();// != null
if (String.IsNullOrWhiteSpace(gline))
{
continue;
}
String[] p = gline.Split(",");
if (p.Length != 5)
{
continue;
}
//push the mapping
globalRegionMap.put(p[2], Int32.Parse(p[0]));
}
greader.Close();
Console.WriteLine("|--[Ok]");
}
//alloc the header size
StreamReader reader = new StreamReader(this.ipSrcFile);
RandomAccessFile raf = new RandomAccessFile(dbFile);
//init the db file
initDbFile(raf);
Console.WriteLine("+-Db file initialized.");
//analysis main loop
Console.WriteLine("+-Try to write the data blocks ... ");
String line = null;
while (!reader.EndOfStream)
{
//line = reader.ReadLine()
line = reader.ReadLine();
if (String.IsNullOrWhiteSpace(line))
{
continue;
}
line = line.Trim();
if (line.Length == 0)
{
continue;
}
if (line[0] == '#')
{
continue;
}
//1. get the start ip
int sIdx = 0, eIdx = 0;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1)
{
continue;
}
String startIp = line.Substring(sIdx, eIdx);
//2. get the end ip
sIdx = eIdx + 1;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1)
{
continue;
}
String endIp = line.Substring(sIdx, eIdx - sIdx);
//3. get the region
sIdx = eIdx + 1;
String region = line.Substring(sIdx);
Console.WriteLine("+-Try to process item " + line);
addDataBlock(raf, startIp, endIp, region);
Console.WriteLine("|--[Ok]");
}
Console.WriteLine("|--Data block flushed!");
Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n");
//write the index bytes
Console.WriteLine("+-Try to write index blocks ... ");
//record the start block
IndexBlock indexBlock = null;
HeaderBlock hb = null;
indexBlock = indexPool.First();
long indexStartIp = indexBlock.getStartIp(),
indexStratPtr = raf.getFilePointer(), indexEndPtr;
headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr)));
int blockLength = IndexBlock.getIndexBlockLength();
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;
//Iterator<IndexBlock> indexIt = indexPool.iterator();
//while ( indexIt.hasNext() ) {
foreach (var block in indexPool)
{
if (++counter >= shotCounter)
{
hb = new HeaderBlock(
block.getStartIp(),
(int)raf.getFilePointer()
);
headerPool.add(hb);
counter = 0;
}
//write the buffer
raf.write(block.getBytes());
}
//record the end block
if (counter > 0)
{
indexBlock = indexPool.Last();
hb = new HeaderBlock(
indexBlock.getStartIp(),
((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength()
);
headerPool.add(hb);
}
indexEndPtr = raf.getFilePointer();
Console.WriteLine("|--[Ok]");
//write the super blocks
Console.WriteLine("+-Try to write the super blocks ... ");
raf.seek(0L); //reset the file pointer
byte[] superBuffer = new byte[8];
Util.writeIntLong(superBuffer, 0, indexStratPtr);
Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength);
raf.write(superBuffer);
Console.WriteLine("|--[Ok]");
//write the header blocks
Console.WriteLine("+-Try to write the header blocks ... ");
//Iterator<HeaderBlock> headerIt = headerPool.iterator();
//while ( headerIt.hasNext() ) {
// HeaderBlock headerBlock = headerIt.next();
// raf.write(headerBlock.getBytes());
//}
foreach (var headerBlock in headerPool)
{
raf.write(headerBlock.getBytes());
}
//write the copyright and the release timestamp info
Console.WriteLine("+-Try to write the copyright and release date info ... ");
raf.seek(raf.length());
//Calendar cal = Calendar.getInstance();
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
var date = DateTime.Now;
String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd");
var timespan = new DateTimeOffset(date).ToUnixTimeSeconds();
raf.write((int)(timespan)); //the unix timestamp
raf.write(copyright.getBytes("UTF-8"));
Console.WriteLine("|--[Ok]");
reader.Close();
raf.close();
}
/**
* construct class
*
* @param bdConfig
* @param dbFile
* @throws FileNotFoundException
*/
public DbSearcher(DbConfig dbConfig, String dbFile)
{
this.dbConfig = dbConfig;
raf = new RandomAccessFile(dbFile);
}