protected override bool RegisterGetPeersMessage(byte[] infoHash, DhtNode node, out TransactionId msgId)
{
var nodeKey = node.CompactEndPoint();
var key = new byte[nodeKey.Length + 2];
Array.Copy(nodeKey, 0, key, 2, nodeKey.Length);
var tryTimes = 0;
while (true)
{
if (tryTimes > 10)
{
break;
}
lock (this)
{
_bucketIndex++;
if (_bucketIndex >= _bucketArray.Length)
{
_bucketIndex = 0;
}
}
msgId = _bucketArray[_bucketIndex];
key[0] = msgId[0];
key[1] = msgId[1];
lock (_syncRoot)
{
var old = _table.Find(msgId);
if (old == null || old.AddTime.AddMinutes(30) < DateTime.Now)
{
_operateSize++;
_table[key] = new Record()
{
InfoHash = infoHash, AddTime = DateTime.Now
};
if (_operateSize > 5120)
{
_storageEngine.Commit();
_operateSize = 0;
}
return(true);
}
}
tryTimes++;
}
msgId = null;
return(false);
}
/// <summary>
/// 保存数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="values"></param>
/// <param name="MarkId"></param>
public static void Save <T>(this List <T> values, string MarkId = "") where T : EntityBase
{
IStorageEngine engine = GetStorageEngine <T>(MarkId);
//var table = engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
var table = engine.GetTable <T>(MarkId); //engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
int iMaxKey = 0;
var selecExist = values.Where(o => o.Sequence > 0);
if (selecExist != null && selecExist.Count() > 0)
{
foreach (T value in selecExist)
{
table[value.Sequence] = value.JsStringify();
}
}
var selecNotExist = values.Where(o => o.Sequence == 0);
if (selecNotExist != null && selecNotExist.Count() > 0)
{
if (table.Count() > 0)
{
iMaxKey = table.Max(o => o.Key);
}
foreach (T value in selecNotExist)
{
//新增
iMaxKey++;
value.Sequence = iMaxKey;
table[value.Sequence] = value.JsStringify();
}
}
engine.Commit();
}
public void SavePage(ref List <Page> docs, string folder, bool isForce = false)
{
// 如果有多余的，需要存放到数据库
lock (obj)
{
if (docs.Count > 50 || isForce)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = folder + "\\Baike_data.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, Page> table = engine.OpenXTable <string, Page>("WebPage");
for (int i = 0; i < docs.Count; i++)
{
if (docs[i].Url.Contains("view"))
{
table[docs[i].Title] = docs[i];
}
}
engine.Commit();
}
docs.Clear();
}
}
}
/// <summary>
/// Create a simple table
/// </summary>
private static void Example1()
{
//insert
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <int, string> table = engine.OpenXTable <int, string>("table1");
for (int i = 0; i < 1000; i++)
{
table[i] = i.ToString();
}
engine.Commit();
}
//read
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <int, string> table = engine.OpenXTable <int, string>("table1");
foreach (var row in table) //table.Forward(), table.Backward()
{
Console.WriteLine("{0} {1}", row.Key, row.Value);
}
}
}
/// <summary>
/// 保存较为正式的网页数据
/// </summary>
/// <param name="docs"></param>
/// <param name="folder"></param>
/// <param name="isForce"></param>
public void SaveContent(ref List <Crawler.Page> docs, string folder, bool isForce = false)
{
// 如果有多余的，需要存放到数据库
lock (PageObj)
{
try
{
if (docs.Count > 50 || isForce)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = folder + "\\Iveely_Search_Engine_Pages.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, Crawler.Page> table = engine.OpenXTable <string, Crawler.Page>("WebPage");
for (int i = 0; i < docs.Count; i++)
{
table[docs[i].Url] = docs[i];
}
engine.Commit();
}
docs.Clear();
}
}
catch (Exception exception)
{
Console.WriteLine(exception);
docs.Clear();
}
}
}
public void DeleteTable(string databaseFileName, string tableName)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFileName))
{
engine.Delete(tableName);
engine.Commit();
}
}
private static void Example(int tickCount, KeysType keysType)
{
Stopwatch sw = new Stopwatch();
const string FILE_NAME = "test.stsdb4";
File.Delete(FILE_NAME);
//insert
Console.WriteLine("Inserting...");
sw.Reset();
sw.Start();
int c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table");
foreach (var kv in TicksGenerator.GetFlow(tickCount, keysType)) //generate random records
{
table[kv.Key] = kv.Value;
c++;
if (c % 100000 == 0)
{
Console.WriteLine("Inserted {0} records", c);
}
}
engine.Commit();
}
sw.Stop();
Console.WriteLine("Insert speed:{0} rec/sec", sw.GetSpeed(tickCount));
//read
Console.WriteLine("Reading...");
sw.Reset();
sw.Start();
c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table");
foreach (var row in table) //table.Forward(), table.Backward()
{
//Console.WriteLine("{0} {1}", row.Key, row.Value);
c++;
if (c % 100000 == 0)
{
Console.WriteLine("Read {0} records", c);
}
}
}
sw.Stop();
Console.WriteLine("Read speed:{0} records", sw.GetSpeed(c));
}
public override void Write(int flowID, IEnumerable <KeyValuePair <long, Tick> > flow)
{
lock (SyncRoot)
{
foreach (var kv in flow)
{
table[kv.Key] = kv.Value;
}
engine.Commit();
}
}
public void Write(IEnumerable <KeyValuePair <long, Tick> > records)
{
lock (SyncRoot)
{
foreach (var record in records)
{
table[record.Key] = record.Value;
}
engine.Commit();
}
}
public void Put(string databaseFile, string tableName, string key, string value)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
table[key] = value;
engine.Commit();
}
Log(databaseFile, tableName);
}
/// <summary>
/// 删除数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <param name="MarkId"></param>
public static void Delete <T>(this T value, string MarkId = "") where T : EntityBase
{
IStorageEngine engine = GetStorageEngine <T>(MarkId);
//var table = engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
var table = engine.GetTable <T>(MarkId); //engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
//var objexist = table.TryGetOrDefault(value.Sequence, "");
//if (!String.IsNullOrWhiteSpace(objexist))//修改
//{
table.Delete(value.Sequence);
//}
engine.Commit();
}
public void Delete(string databaseFile, string tableName, string key)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
table.Delete(key);
engine.Commit();
}
Log(databaseFile, tableName);
}
private void InsertEntity(IEnumerable <KnowledgeEntity> ces)
{
string dataPath = "Baike\\Baike_question.db4";
using (IStorageEngine engine = STSdb.FromFile(dataPath))
{
ITable <long, KnowledgeEntity> table = engine.OpenXTable <long, KnowledgeEntity>("WebPage");
foreach (var knowledgeEntity in ces)
{
table[knowledgeEntity.Id] = knowledgeEntity;
InsertIndex(knowledgeEntity.Id, knowledgeEntity.QuestionDesc);
}
engine.Commit();
}
}
/// <summary>
/// 保存索引数据
/// </summary>
/// <param name="indexDocs"></param>
/// <param name="folder"></param>
/// <param name="isForce"></param>
public void SaveIndex(ref List <TextIndex> indexDocs, string folder, bool isForce = false)
{
// 如果有多余的，需要存放到数据库
lock (IndexObj)
{
try
{
if (indexDocs.Count > 2000 || isForce)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = folder + "\\Iveely_Search_Engine_TextIndex.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, List <Iveely.STSdb4.Data.Slots <string, double> > > table = engine.OpenXTable <string, List <Iveely.STSdb4.Data.Slots <string, double> > >("WebPage");
for (int i = 0; i < indexDocs.Count; i++)
{
// 如果包含则追加
List <Iveely.STSdb4.Data.Slots <string, double> > list = table.Find(indexDocs[i].Keyword);
if (list != null && list.Count > 0)
{
Iveely.STSdb4.Data.Slots <string, double> slot = new Slots <string, double>(indexDocs[i].Url, indexDocs[i].Weight);
list.Add(slot);
}
// 否则新增
else
{
list = new List <Slots <string, double> >();
Iveely.STSdb4.Data.Slots <string, double> slot = new Slots <string, double>(indexDocs[i].Url, indexDocs[i].Weight);
list.Add(slot);
table[indexDocs[i].Keyword] = list;
}
}
engine.Commit();
}
indexDocs.Clear();
}
}
catch (Exception exception)
{
Console.WriteLine(exception);
indexDocs.Clear();
}
}
}
private void InsertIndex(long id, string text)
{
string dataPath = "Baike\\Baike_question_index.db4";
var frequency = new IntTable <string, int>();
string[] results = _segment.Split(text);
if (results.Length < 1)
{
return;
}
frequency.Add(results);
foreach (DictionaryEntry de in frequency)
{
KeywordIndex keywordIndex = new KeywordIndex();
keywordIndex.Keyword = de.Key.ToString();
keywordIndex.Weight = int.Parse(de.Value.ToString()) * 1.0 / results.Length;
keywordIndex.Id = id;
Indexs.Add(keywordIndex);
}
if (Indexs.Count > 0)
{
using (IStorageEngine engine = STSdb.FromFile(dataPath))
{
ITable <string, List <Slots <long, double> > > table = engine.OpenXTable <string, List <Slots <long, double> > >("WebPage");
foreach (var keywordIndex in Indexs)
{
// 如果包含则追加
List <Slots <long, double> > list = table.Find(keywordIndex.Keyword);
if (list != null && list.Count > 0)
{
Slots <long, double> slot = new Slots <long, double>(keywordIndex.Id, keywordIndex.Weight);
list.Add(slot);
}
// 否则新增
else
{
list = new List <Slots <long, double> >();
Slots <long, double> slot = new Slots <long, double>(keywordIndex.Id, keywordIndex.Weight);
list.Add(slot);
table[keywordIndex.Keyword] = list;
}
}
engine.Commit();
}
Indexs.Clear();
}
}
/// <summary>
/// 删除数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="values"></param>
/// <param name="MarkId"></param>
public static void Delete <T>(this List <T> values, string MarkId = "") where T : EntityBase
{
IStorageEngine engine = GetStorageEngine <T>(MarkId);
//var table = engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
var table = engine.GetTable <T>(MarkId); //engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
//var objexist = table.TryGetOrDefault(value.Sequence, "");
//if (!String.IsNullOrWhiteSpace(objexist))//修改
//{
// table.Delete(objexist.JsParse<T>().Sequence);
//}
foreach (T value in values)
{
table.Delete(value.Sequence);
System.Threading.Thread.Sleep(10);
}
engine.Commit();
}
/// <summary>
/// 保存数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <param name="MarkId"></param>
public static void Save <T>(this T value, string MarkId = "") where T : EntityBase
{
IStorageEngine engine = GetStorageEngine <T>(MarkId);
//var table = engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
var table = engine.GetTable <T>(MarkId); //engine.OpenXTable<int, string>(typeof(T).Name + MarkId);
int iMaxKey = 0;
if (value.Sequence == 0)
{
if (table.Count() > 0)
{
iMaxKey = table.Max(o => o.Key);
}
//新增
iMaxKey++;
value.Sequence = iMaxKey;
}
table[value.Sequence] = value.JsStringify();
engine.Commit();
}
/// <summary>
/// Create a table with user type
/// </summary>
private static void Example2()
{
Random random = new Random();
//insert
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table2");
for (int i = 0; i < 1000; i++)
{
long key = random.Next();
Tick tick = new Tick();
tick.Type = TickType.Forex;
tick.Symbol = "";
tick.Timestamp = DateTime.Now;
tick.Bid = Math.Round(random.NextDouble(), 4);
tick.Ask = Math.Round(tick.Bid + 0.0001 * random.Next(2, 10), 4);
tick.Volume = i;
tick.Provider = "";
table[key] = tick;
}
engine.Commit();
}
////read
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table2");
foreach (var row in table) //table.Forward(), table.Backward()
{
Console.WriteLine("{0} {1}", row.Key, row.Value);
}
}
}
private void PacketExecute(object state)
{
try
{
KeyValuePair <ServerConnection, Packet> order = (KeyValuePair <ServerConnection, Packet>)state;
BinaryReader reader = new BinaryReader(order.Value.Request);
Message msgRequest = Message.Deserialize(reader);
MemoryStream ms = new MemoryStream();
BinaryWriter writer = new BinaryWriter(ms);
IOperationCollection asyncOperations = new OperationCollection(msgRequest.Locator, msgRequest.Operations.Count);
IOperationCollection resultsOperations = new OperationCollection(msgRequest.Locator, 1);
var index = StorageEngine.OpenXIndex(msgRequest.Locator.KeyDescriptor, msgRequest.Locator.RecordDescriptor, msgRequest.Locator.Name);
foreach (var operation in msgRequest.Operations)
{
try
{
if (!operation.IsSynchronous)
{
asyncOperations.Add(operation);
}
else
{
StorageEngine.Execute(asyncOperations);
asyncOperations.Clear();
switch (operation.Code)
{
case OperationCode.TRY_GET:
{
IData record = null;
index.TryGet(operation.FromKey, out record);
resultsOperations.Add(new TryGetOperation(operation.FromKey, record));
}
break;
case OperationCode.FORWARD:
{
var op = (ForwardOperation)operation;
List <KeyValuePair <IData, IData> > forward = index.Forward(op.FromKey, op.FromKey != null, op.ToKey, op.ToKey != null).Take(op.PageCount).ToList();
ForwardOperation opResult = new ForwardOperation(0, op.FromKey, op.ToKey, forward);
resultsOperations.Add(opResult);
}
break;
case OperationCode.BACKWARD:
{
var op = (BackwardOperation)operation;
List <KeyValuePair <IData, IData> > backward = index.Backward(op.FromKey, op.FromKey != null, op.ToKey, op.ToKey != null).Take(op.PageCount).ToList();
BackwardOperation opResult = new BackwardOperation(0, op.FromKey, op.ToKey, backward);
resultsOperations.Add(opResult);
}
break;
case OperationCode.COUNT:
{
resultsOperations.Add(new CountOperation(index.Count()));
}
break;
case OperationCode.FIND_NEXT:
{
var kv = index.FindNext(operation.FromKey);
resultsOperations.Add(new FindNextOperation(operation.FromKey, kv));
}
break;
case OperationCode.FIND_AFTER:
{
var kv = index.FindAfter(operation.FromKey);
resultsOperations.Add(new FindAfterOperation(operation.FromKey, kv));
}
break;
case OperationCode.FIND_PREV:
{
var kv = index.FindPrev(operation.FromKey);
resultsOperations.Add(new FindPrevOperation(operation.FromKey, kv));
}
break;
case OperationCode.FIND_BEFORE:
{
var kv = index.FindBefore(operation.FromKey);
resultsOperations.Add(new FindBeforeOperation(operation.FromKey, kv));
}
break;
case OperationCode.FIRST_ROW:
{
var kv = index.FirstRow;
resultsOperations.Add(new FirstRowOperation(kv));
}
break;
case OperationCode.LAST_ROW:
{
var kv = index.LastRow;
resultsOperations.Add(new LastRowOperation(kv));
}
break;
case OperationCode.STORAGE_ENGINE_COMMIT:
{
StorageEngine.Commit();
resultsOperations.Add(new StorageEngineCommitOperation());
}
break;
}
}
}
catch (Exception exc)
{
resultsOperations.Add(new ExceptionOperation(exc.Message));
}
}
if (asyncOperations.Count > 0)
{
StorageEngine.Execute(asyncOperations);
}
index.Flush();
Message msgResponse = new Message(resultsOperations);
msgResponse.Serialize(writer);
ms.Position = 0;
order.Value.Response = ms;
order.Key.PendingPackets.Add(order.Value);
}
catch (Exception exc)
{
TcpServer.LogError(exc);
}
}
public static void Initialize()
{
_engine = STSdb.FromFile(Config.Db);
var regions = _engine.OpenXTable<int, AOCRegion>("regions");
var organisations = _engine.OpenXTable<int, AOCOrganisation>("organisations");
var users = _engine.OpenXTable<string, AOCUser>("users");
foreach (var o in Config.Organisations)
{
AOCOrganisation org = new AOCOrganisation()
{
ID = StringComparer.OrdinalIgnoreCase.GetHashCode(o),
Name = o
};
organisations[org.ID] = org;
}
Console.WriteLine("Added {0} organisations", organisations.Count());
foreach (var r in Config.Regions)
{
AOCRegion reg = new AOCRegion()
{
ID = StringComparer.OrdinalIgnoreCase.GetHashCode(r),
Name = r
};
regions[reg.ID] = reg;
}
Console.WriteLine("Added {0} regions", regions.Count());
foreach (var u in Config.Users)
{
users[u.Email] = u;
}
Console.WriteLine("Added {0} users", users.Count());
#if DEBUG
var entries = _engine.OpenXTable<string, AOCEntry>("entries");
var names = new Dictionary<string, string>()
{
{ "Julian", "Vallis"},
{ "Tim", "Peet" },
{ "Emma", "Baines" },
{ "Rachel", "Gaskell" },
{ "Karen", "Porter" },
{ "Matt", "Cridland" },
{ "Ross", "Hopcraft" },
{ "Helene", "Dickson" },
{ "April", "Felker" },
{ "Fiona", "Vallis" },
{ "Sandra", "Lim" },
{ "Bilbo", "Baggins" },
{ "Jeremy", "Paxman" },
{ "Freddie", "Mercury" },
{ "Mick", "Jagger" },
{ "Keith", "Richards" },
{ "Bill", "Wyman" },
{ "Charlie", "Watts" },
{ "Carlos", "Santana" },
{ "Paul", "McCartney" },
{ "John", "Lennon" },
{ "James", "Bond" },
{ "Sean", "Connery" },
{ "George", "Lazenby" },
{ "Roger", "Moore" },
{ "Timothy", "Dalton" },
{ "Pierce", "Brosnan" },
{ "Daniel", "Craig" },
};
foreach (var name in names)
{
entries[name.Key.ToLowerInvariant() + "@bisqit.co.uk"] = new AOCEntry()
{
Answers = "1,2,3,4,1,2,3,4",
Available = true,
Email = name.Key.ToLowerInvariant() + "@bisqit.co.uk",
FirstName = name.Key,
LastName = name.Value,
LineManager = "God",
Organisation = "Bisqit",
Reason = "Because...",
Region = "London",
Role = "God",
Score = 10,
Submitted = DateTime.UtcNow
};
}
#endif
_engine.Commit();
}
private ICommand StorageEngineCommit(ICommand command)
{
StorageEngine.Commit();
return(new StorageEngineCommitCommand());
}
public void Commit()
{
_storageEngine.Commit();
}
public void Commit()
{
Engine.Commit();
}
