public void Start()
{
string dataPath = "Baike\\Baike_data.db4";
using (IStorageEngine engine = STSdb.FromFile(dataPath))
{
// 1.提取数据
ITable <string, BaikeDataCrawler.Page> table = engine.OpenXTable <string, BaikeDataCrawler.Page>("WebPage");
long totalCount = table.Count();
foreach (var keyValuePair in table)
{
Console.WriteLine(totalCount--);
BaikeDataCrawler.Page page = keyValuePair.Value;
// 2.提取问题
List <QuestionGetter.QuestionEntity> questionEntities = _questionGetter.GetKnowledge(page.Content);
if (questionEntities != null && questionEntities.Count > 0)
{
foreach (var questionEntity in questionEntities)
{
KnowledgeEntity entity = new KnowledgeEntity();
entity.EntityA = questionEntity.EntityA;
entity.EntityB = questionEntity.EntityB;
entity.QuestionDesc = questionEntity.QuestionDesc;
entity.Answer = questionEntity.Answer;
entity.Relation = questionEntity.Relation;
entity.EffectTime = DateTime.Now.ToShortDateString();
entity.RefUrl = page.Url;
entity.Id = _currentId++;
entities.Add(entity);
}
}
// 3.存储数据
if (entities.Count > Maxcount)
{
IEnumerable <KnowledgeEntity> ces = entities.Distinct();
if (ces.Count() > 0)
{
InsertEntity(ces);
}
entities.Clear();
}
}
}
if (entities != null && entities.Count > 0)
{
IEnumerable <KnowledgeEntity> ces = entities.Distinct();
if (ces.Count() > 0)
{
InsertEntity(ces);
}
}
Console.ReadLine();
}
/// <summary>
/// 更新
/// </summary>
/// <param name="pEntity">实体实例</param>
/// <param name="pTran">事务实例,可为null,如果为null,则不使用事务来更新</param>
public void Update(KnowledgeEntity pEntity, IDbTransaction pTran)
{
_currentDAO.Update(pEntity, pTran);
}
/// <summary>
/// 创建一个新实例
/// </summary>
/// <param name="pEntity">实体实例</param>
public void Create(KnowledgeEntity pEntity)
{
_currentDAO.Create(pEntity);
}
/// <summary>
/// 分页根据实体条件查询实体
/// </summary>
/// <param name="pQueryEntity">以实体形式传入的参数</param>
/// <param name="pOrderBys">排序组合</param>
/// <returns>符合条件的实体集</returns>
public PagedQueryResult <KnowledgeEntity> PagedQueryByEntity(KnowledgeEntity pQueryEntity, OrderBy[] pOrderBys, int pPageSize, int pCurrentPageIndex)
{
return(_currentDAO.PagedQueryByEntity(pQueryEntity, pOrderBys, pPageSize, pCurrentPageIndex));
}
/// <summary>
/// 根据实体条件查询实体
/// </summary>
/// <param name="pQueryEntity">以实体形式传入的参数</param>
/// <param name="pOrderBys">排序组合</param>
/// <returns>符合条件的实体集</returns>
public KnowledgeEntity[] QueryByEntity(KnowledgeEntity pQueryEntity, OrderBy[] pOrderBys)
{
return(_currentDAO.QueryByEntity(pQueryEntity, pOrderBys));
}
/// <summary>
/// 删除
/// </summary>
/// <param name="pEntity"></param>
public void Delete(KnowledgeEntity pEntity)
{
_currentDAO.Delete(pEntity);
}
/// <summary>
/// 更新
/// </summary>
/// <param name="pEntity">实体实例</param>
public void Update(KnowledgeEntity pEntity)
{
_currentDAO.Update(pEntity);
}
