/// <summary>
/// Create a simple table
/// </summary>
private static void Example1()
{
//insert
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <int, string> table = engine.OpenXTable <int, string>("table1");
for (int i = 0; i < 1000; i++)
{
table[i] = i.ToString();
}
engine.Commit();
}
//read
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <int, string> table = engine.OpenXTable <int, string>("table1");
foreach (var row in table) //table.Forward(), table.Backward()
{
Console.WriteLine("{0} {1}", row.Key, row.Value);
}
}
}
public void SavePage(ref List <Page> docs, string folder, bool isForce = false)
{
// 如果有多余的，需要存放到数据库
lock (obj)
{
if (docs.Count > 50 || isForce)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = folder + "\\Baike_data.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, Page> table = engine.OpenXTable <string, Page>("WebPage");
for (int i = 0; i < docs.Count; i++)
{
if (docs[i].Url.Contains("view"))
{
table[docs[i].Title] = docs[i];
}
}
engine.Commit();
}
docs.Clear();
}
}
}
/// <summary>
/// 保存较为正式的网页数据
/// </summary>
/// <param name="docs"></param>
/// <param name="folder"></param>
/// <param name="isForce"></param>
public void SaveContent(ref List <Crawler.Page> docs, string folder, bool isForce = false)
{
// 如果有多余的，需要存放到数据库
lock (PageObj)
{
try
{
if (docs.Count > 50 || isForce)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = folder + "\\Iveely_Search_Engine_Pages.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, Crawler.Page> table = engine.OpenXTable <string, Crawler.Page>("WebPage");
for (int i = 0; i < docs.Count; i++)
{
table[docs[i].Url] = docs[i];
}
engine.Commit();
}
docs.Clear();
}
}
catch (Exception exception)
{
Console.WriteLine(exception);
docs.Clear();
}
}
}
static void Main(string[] args)
{
try
{
using (IStorageEngine engine = STSdb.FromFile("Iveely.db"))
{
var server = STSdb.CreateServer(engine, 7182);
server.Start();
while (true)
{
string cmd = Console.ReadLine();
if (cmd == "exit")
{
break;
}
else
{
Console.WriteLine("Enter 'exit' to exit program...");
}
}
}
}
catch (Exception exception)
{
Console.WriteLine(exception);
}
}
public DatabaseContainer(string FileName)
{
Engine = STSdb.FromFile(Path.Combine(Environment.CurrentDirectory, FileName));
Students = Engine.OpenXTable <int, Student>("Student");
Disciplines = Engine.OpenXTable <int, Discipline>("Disciplines");
Chapters = Engine.OpenXTable <int, Chapter>("Chapters");
Questions = Engine.OpenXTable <int, Question>("Questions");
}
private static void LoadRawData()
{
string dataPath = "TextSearch\\ISE_Pages.db4";
IStorageEngine dataEngine = STSdb.FromFile(dataPath);//.OpenXFile(indexFilePath);
dataEngine.OpenXFile(dataPath);
SearchDataTable = dataEngine.OpenXTable <string, Crawler.Page>("WebPage");
}
private static void LoadKnowledgeData()
{
string dataPath = "Baike\\Baike_question.db4";
IStorageEngine dataEngine = STSdb.FromFile(dataPath);
dataEngine.OpenXFile(dataPath);
KnowledgeDataTable = dataEngine.OpenXTable <long, KnowlegeIndex.KnowledgeEntity>("WebPage");
}
public bool Exists(string databaseFile, string tableName, string key)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
return(table.Exists(key));
}
}
public void DeleteTable(string databaseFileName, string tableName)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFileName))
{
engine.Delete(tableName);
engine.Commit();
}
}
private static void Example(int tickCount, KeysType keysType)
{
Stopwatch sw = new Stopwatch();
const string FILE_NAME = "test.stsdb4";
File.Delete(FILE_NAME);
//insert
Console.WriteLine("Inserting...");
sw.Reset();
sw.Start();
int c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table");
foreach (var kv in TicksGenerator.GetFlow(tickCount, keysType)) //generate random records
{
table[kv.Key] = kv.Value;
c++;
if (c % 100000 == 0)
{
Console.WriteLine("Inserted {0} records", c);
}
}
engine.Commit();
}
sw.Stop();
Console.WriteLine("Insert speed:{0} rec/sec", sw.GetSpeed(tickCount));
//read
Console.WriteLine("Reading...");
sw.Reset();
sw.Start();
c = 0;
using (IStorageEngine engine = STSdb.FromFile(FILE_NAME))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table");
foreach (var row in table) //table.Forward(), table.Backward()
{
//Console.WriteLine("{0} {1}", row.Key, row.Value);
c++;
if (c % 100000 == 0)
{
Console.WriteLine("Read {0} records", c);
}
}
}
sw.Stop();
Console.WriteLine("Read speed:{0} records", sw.GetSpeed(c));
}
public void Start()
{
string dataPath = "Baike\\Baike_data.db4";
using (IStorageEngine engine = STSdb.FromFile(dataPath))
{
// 1.提取数据
ITable <string, BaikeDataCrawler.Page> table = engine.OpenXTable <string, BaikeDataCrawler.Page>("WebPage");
long totalCount = table.Count();
foreach (var keyValuePair in table)
{
Console.WriteLine(totalCount--);
BaikeDataCrawler.Page page = keyValuePair.Value;
// 2.提取问题
List <QuestionGetter.QuestionEntity> questionEntities = _questionGetter.GetKnowledge(page.Content);
if (questionEntities != null && questionEntities.Count > 0)
{
foreach (var questionEntity in questionEntities)
{
KnowledgeEntity entity = new KnowledgeEntity();
entity.EntityA = questionEntity.EntityA;
entity.EntityB = questionEntity.EntityB;
entity.QuestionDesc = questionEntity.QuestionDesc;
entity.Answer = questionEntity.Answer;
entity.Relation = questionEntity.Relation;
entity.EffectTime = DateTime.Now.ToShortDateString();
entity.RefUrl = page.Url;
entity.Id = _currentId++;
entities.Add(entity);
}
}
// 3.存储数据
if (entities.Count > Maxcount)
{
IEnumerable <KnowledgeEntity> ces = entities.Distinct();
if (ces.Count() > 0)
{
InsertEntity(ces);
}
entities.Clear();
}
}
}
if (entities != null && entities.Count > 0)
{
IEnumerable <KnowledgeEntity> ces = entities.Distinct();
if (ces.Count() > 0)
{
InsertEntity(ces);
}
}
Console.ReadLine();
}
public Storage(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentException("fileName");
}
this.FileName = fileName;
this.storageEngine = STSdb.FromFile(this.FileName);
}
public IEnumerable <KeyValuePair <string, string> > GetAll(string databaseFile, string tableName)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
foreach (var row in table)
{
yield return(row);
}
}
}
public void Put(string databaseFile, string tableName, string key, string value)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
table[key] = value;
engine.Commit();
}
Log(databaseFile, tableName);
}
/// <summary>
/// Create server
/// </summary>
private static void Example4()
{
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
var server = STSdb.CreateServer(engine, 7182);
server.Start();
//server is ready for connections
server.Stop();
}
}
public void Delete(string databaseFile, string tableName, string key)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
table.Delete(key);
engine.Commit();
}
Log(databaseFile, tableName);
}
private void Log(string databaseFile, string tableName)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
log.Info(string.Format("Rows in {0}.{1}:", databaseFile, tableName));
foreach (var row in table)
{
log.Info(string.Format("{0} => {1}", row.Key, row.Value));
}
}
}
public void Index()
{
string fileFlag = GetRootFolder() + "\\RawData\\Baike_data.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, Page> table = engine.OpenXTable <string, Page>("WebPage");
foreach (var kv in table)
{
Page page = (Page)kv.Value;
Console.WriteLine(kv.Key + " " + page.Url);
}
}
}
private void InsertEntity(IEnumerable <KnowledgeEntity> ces)
{
string dataPath = "Baike\\Baike_question.db4";
using (IStorageEngine engine = STSdb.FromFile(dataPath))
{
ITable <long, KnowledgeEntity> table = engine.OpenXTable <long, KnowledgeEntity>("WebPage");
foreach (var knowledgeEntity in ces)
{
table[knowledgeEntity.Id] = knowledgeEntity;
InsertIndex(knowledgeEntity.Id, knowledgeEntity.QuestionDesc);
}
engine.Commit();
}
}
/// <summary>
/// 保存索引数据
/// </summary>
/// <param name="indexDocs"></param>
/// <param name="folder"></param>
/// <param name="isForce"></param>
public void SaveIndex(ref List <TextIndex> indexDocs, string folder, bool isForce = false)
{
// 如果有多余的，需要存放到数据库
lock (IndexObj)
{
try
{
if (indexDocs.Count > 2000 || isForce)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = folder + "\\Iveely_Search_Engine_TextIndex.db4";
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, List <Iveely.STSdb4.Data.Slots <string, double> > > table = engine.OpenXTable <string, List <Iveely.STSdb4.Data.Slots <string, double> > >("WebPage");
for (int i = 0; i < indexDocs.Count; i++)
{
// 如果包含则追加
List <Iveely.STSdb4.Data.Slots <string, double> > list = table.Find(indexDocs[i].Keyword);
if (list != null && list.Count > 0)
{
Iveely.STSdb4.Data.Slots <string, double> slot = new Slots <string, double>(indexDocs[i].Url, indexDocs[i].Weight);
list.Add(slot);
}
// 否则新增
else
{
list = new List <Slots <string, double> >();
Iveely.STSdb4.Data.Slots <string, double> slot = new Slots <string, double>(indexDocs[i].Url, indexDocs[i].Weight);
list.Add(slot);
table[indexDocs[i].Keyword] = list;
}
}
engine.Commit();
}
indexDocs.Clear();
}
}
catch (Exception exception)
{
Console.WriteLine(exception);
indexDocs.Clear();
}
}
}
private static void LoadTextIndex()
{
string indexFilePath = "TextSearch\\ISE_Pages_Index.db4";
using (IStorageEngine textIndexEngine = STSdb.FromFile(indexFilePath))
{
ITable <string, List <Iveely.STSdb4.Data.Slots <string, double> > > indexTable = textIndexEngine.OpenXTable <string, List <Iveely.STSdb4.Data.Slots <string, double> > >("WebPage");
foreach (var kv in indexTable)
{
if (!TextIndexData.ContainsKey(kv.Key))
{
TextIndexData[kv.Key] = kv.Value;
}
}
}
}
private static void LoadKnowledgeIndex()
{
string indexFilePath = "Baike\\Baike_question_index.db4";
using (IStorageEngine textIndexEngine = STSdb.FromFile(indexFilePath))
{
ITable <string, List <Iveely.STSdb4.Data.Slots <long, double> > > indexTable = textIndexEngine.OpenXTable <string, List <Iveely.STSdb4.Data.Slots <long, double> > >("WebPage");
foreach (var kv in indexTable)
{
if (!KnowledgeIndexData.ContainsKey(kv.Key))
{
KnowledgeIndexData[kv.Key] = kv.Value;
}
}
}
}
public string Get(string databaseFile, string tableName, string key)
{
using (IStorageEngine engine = STSdb.FromFile(databaseFile))
{
var table = engine.OpenXTable <string, string>(tableName);
if (table.Exists(key))
{
return(table[key]);
}
else
{
return(null);
}
}
}
private void InsertIndex(long id, string text)
{
string dataPath = "Baike\\Baike_question_index.db4";
var frequency = new IntTable <string, int>();
string[] results = _segment.Split(text);
if (results.Length < 1)
{
return;
}
frequency.Add(results);
foreach (DictionaryEntry de in frequency)
{
KeywordIndex keywordIndex = new KeywordIndex();
keywordIndex.Keyword = de.Key.ToString();
keywordIndex.Weight = int.Parse(de.Value.ToString()) * 1.0 / results.Length;
keywordIndex.Id = id;
Indexs.Add(keywordIndex);
}
if (Indexs.Count > 0)
{
using (IStorageEngine engine = STSdb.FromFile(dataPath))
{
ITable <string, List <Slots <long, double> > > table = engine.OpenXTable <string, List <Slots <long, double> > >("WebPage");
foreach (var keywordIndex in Indexs)
{
// 如果包含则追加
List <Slots <long, double> > list = table.Find(keywordIndex.Keyword);
if (list != null && list.Count > 0)
{
Slots <long, double> slot = new Slots <long, double>(keywordIndex.Id, keywordIndex.Weight);
list.Add(slot);
}
// 否则新增
else
{
list = new List <Slots <long, double> >();
Slots <long, double> slot = new Slots <long, double>(keywordIndex.Id, keywordIndex.Weight);
list.Add(slot);
table[keywordIndex.Keyword] = list;
}
}
engine.Commit();
}
Indexs.Clear();
}
}
/// <summary>
/// 分析原始网页数据
/// </summary>
/// <param name="folder"></param>
public void AnalysisData(string folder, string filePath)
{
// 如果有多余的，需要存放到数据库
lock (PageObj)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
string fileFlag = filePath;
using (IStorageEngine engine = STSdb.FromFile(fileFlag))
{
// 插入数据
ITable <string, Crawler.Page> table = engine.OpenXTable <string, Crawler.Page>("WebPage");
foreach (var keyValuePair in table)
{
Crawler.Page page = (Crawler.Page)keyValuePair.Value;
if (page != null && page.Content.Trim().Length > 0)
{
Console.WriteLine(page.Url);
var frequency = new IntTable <string, int>();
string[] results = segment.Split(page.Title + page.Title + page.Content);
if (results.Length < 1)
{
continue;
}
frequency.Add(results);
foreach (DictionaryEntry de in frequency)
{
TextIndex textIndex = new TextIndex();
textIndex.Keyword = de.Key.ToString();
textIndex.Weight = int.Parse(de.Value.ToString()) * 1.0 / results.Length;
textIndex.Url = page.Url;
indexs.Add(textIndex);
SaveIndex(ref indexs, folder, false);
}
pages.Add(page);
SaveContent(ref pages, folder, false);
}
}
}
}
SaveIndex(ref indexs, folder, true);
SaveContent(ref pages, folder, true);
}
protected override void OnStart(string[] args)
{
string FileName = ConfigurationSettings.AppSettings["FileName"];
int port = int.Parse(ConfigurationSettings.AppSettings["Port"]);
int boundedCapacity = int.Parse(ConfigurationSettings.AppSettings["BoundedCapacity"]);
Service = this;
StorageEngine = STSdb.FromFile(FileName);
TcpServer = new TcpServer(port);
Program.StorageEngineServer = new StorageEngineServer(StorageEngine, TcpServer);
Program.StorageEngineServer.Start();
Form = new MainForm();
Application.Run(Form);
Program.StorageEngineServer.Stop();
StorageEngine.Close();
}
/// <summary>
/// 获取存储引擎
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="MarkId"></param>
/// <returns></returns>
private static IStorageEngine GetStorageEngine <T>(string MarkId = "")
{
IStorageEngine engine = null;
string strKey = typeof(T).Name;
var engineData = new KeyValuePair <string, IStorageEngine>();
lock (Engines)
{
engineData = Engines.FirstOrDefault(o => o.Key == strKey + MarkId);
}
if (engineData.Value == null)
{
var dbFile = GetDB <T>(MarkId);
try
{
//engine = STSdb.FromStream(System.IO.File.Open(dbFile, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite));
engine = STSdb.FromFile(dbFile);
//engine = STSdb.FromMemory();
var table = engine.OpenXTable <int, string>(strKey);
table.FirstOrDefault();
}
catch (Exception ex)
{
("数据库出错，异常信息：" + ex.ToString()).WriteToLog(log4net.Core.Level.Error);
engine.Heap.Close();
System.Threading.Thread.Sleep(100);
System.IO.File.Move(dbFile, (dbFile + "_" + DateTime.Now.ToString("yyyyMMhhddmmss") + ".bak"));
System.Threading.Thread.Sleep(100);
engine = STSdb.FromFile(dbFile);
}
lock (Engines)
{
Engines.Add(strKey + MarkId, engine);
}
}
else
{
engine = engineData.Value;
}
return(engine);
}
/// <summary>
/// Create a table with user type
/// </summary>
private static void Example2()
{
Random random = new Random();
//insert
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table2");
for (int i = 0; i < 1000; i++)
{
long key = random.Next();
Tick tick = new Tick();
tick.Type = TickType.Forex;
tick.Symbol = "";
tick.Timestamp = DateTime.Now;
tick.Bid = Math.Round(random.NextDouble(), 4);
tick.Ask = Math.Round(tick.Bid + 0.0001 * random.Next(2, 10), 4);
tick.Volume = i;
tick.Provider = "";
table[key] = tick;
}
engine.Commit();
}
////read
using (IStorageEngine engine = STSdb.FromFile("test.stsdb4"))
{
ITable <long, Tick> table = engine.OpenXTable <long, Tick>("table2");
foreach (var row in table) //table.Forward(), table.Backward()
{
Console.WriteLine("{0} {1}", row.Key, row.Value);
}
}
}
private static void Test()
{
int count = 20000000;
string fileName = "test.stsdb4";
//File.Delete(fileName);
Random random = new Random();
Stopwatch sw = new Stopwatch();
double progress = 0.0;
//insert
Console.WriteLine(String.Format("Inserting {0} records...", count));
sw.Start();
using (var engine = STSdb.FromFile(fileName))
{
var index = engine.OpenXTable <int, int>("table");
for (int i = 0; i < count; i++)
{
int key = random.Next(0, int.MaxValue);
int rec = i;
index[key] = rec;
double p = Math.Round(100.0 * (i + 1) / count);
if (p - progress >= 5)
{
Console.Write(String.Format("{0}% ", p));
progress = p;
}
}
engine.Commit();
Console.WriteLine("{0} rec/sec", sw.GetSpeed(count));
Console.WriteLine("~{0}MB", Math.Round(engine.DatabaseSize / (1024 * 1024.0)));
}
sw.Stop();
progress = 0;
//read
Console.WriteLine("Reading...");
sw.Reset();
sw.Start();
int c = 0;
using (var engine = STSdb.FromFile(fileName))
{
var index = engine.OpenXTable <int, int>("table");
int key = -1;
foreach (var kv in index)
{
if (key > kv.Key)
{
throw new Exception();
}
key = kv.Key;
c++;
double p = Math.Round(100.0 * c / count);
if (p - progress >= 5)
{
Console.Write(String.Format("{0}% ", p));
progress = p;
}
}
Console.WriteLine(String.Format("{0} records", c));
Console.WriteLine("{0} rec/sec", sw.GetSpeed(count));
Console.WriteLine("~{0}MB", Math.Round(engine.DatabaseSize / (1024 * 1024.0)));
}
sw.Stop();
Thread.Sleep(100000);
}
public STSdbContext()
{
this.Context = STSdb.FromFile(GetDatabaseFile());
}