protected override List <EntityCase> HandleCore(IndexEntity message)
{
var department = _db.EntityCase
.Where(d => (message.ID == null) || d.Id == message.ID).ToList <EntityCase>();
return(department);
}
public string GetRecentUrlsCrawled(int n)
{
List <string> results = new List <string>();
TableQuery <IndexEntity> query = new TableQuery <IndexEntity>();
int i = 0;
foreach (IndexEntity entity in indexTable.ExecuteQuery(query))
{
if (i == n)
{
break;
}
i++;
results.Add(IndexEntity.Base64Decode(entity.PartitionKey));
}
if (results.Count == 0)
{
results.Add("Index is empty");
}
return(new JavaScriptSerializer().Serialize(results));
}
private async Task createIndexIfNotExist()
{
long ret = await getIndex();
if (ret == -1)
{
IndexEntity i = new IndexEntity(_version);
i.index = 0;
TableOperation insertOperation = TableOperation.Insert(i);
await _indextable.ExecuteAsync(insertOperation);
}
}
public void Crawl(string url)
{
if (!visitedUrls.ContainsKey(url))
{
HtmlDocument document;
try {
document = web.Load(url);
} catch
{
Logger.Instance.Log(Logger.LOG_ERROR, "html dom parsing failed in WebCrawler.Crawl() for " + url);
return;
}
Link parentLink = new Link(url);
HtmlNode[] nodes = document.DocumentNode.SelectNodes("//title").ToArray();
string title = "Page indexed, but no <title> tag found";
foreach (HtmlNode item in nodes)
{
title = item.InnerHtml;
break; // there should only be one title, if there are more then pick the 1st one arbitrarily
}
IndexEntity indexEntity = new IndexEntity(url, title); // also add page date
TableOperation insertOperation = TableOperation.Insert(indexEntity);
urlTable.Execute(insertOperation);
visitedUrls.Add(url, true);
nodes = document.DocumentNode.SelectNodes("//a").ToArray();
foreach (HtmlNode item in nodes)
{
string linkPath = item.GetAttributeValue("href", ""); // could be relative or absolute or external
string link = parentLink.buildUrl(linkPath);
if (!visitedUrls.ContainsKey(link) && urlValidator.IsUrlValidCrawling(link))
{
visitedUrls.Add(link, true);
UrlMessage urlEntity = new UrlMessage(UrlMessage.URL_TYPE_HTML, link);
// Add message
CloudQueueMessage message = new CloudQueueMessage(urlEntity.ToString());
urlQueue.AddMessage(message);
}
}
}
}
public async Task <bool> Init()
{
var table = await TableExecutor <TableEntity> .CreateTableAsync("shorturls", _cloudStorage);
for (int i = 0; i < 99; i++)
{
var partkey = i.ToString("00");
var index = await TableExecutor <IndexEntity> .PointQueryAsync(table, partkey, "index");
if (index == null)
{
index = new IndexEntity(partkey, "index", 0L);
await TableExecutor <IndexEntity> .InsertOrMergeEntityAsync(table, index);
}
TableIndex.AddOrUpdate(partkey, index.Index, (k, v) => v = index.Index);
}
return(true);
}
public async Task <long> createIndexIfNotExist(string version)
{
long ret = await getIndex(version);
if (ret == -1)
{
IndexEntity i = new IndexEntity(version);
i.index = 0;
TableOperation insertOperation = TableOperation.Insert(i);
await _indextable.ExecuteAsync(insertOperation);
Console.WriteLine("Created index for version " + version);
return(0);
}
Console.WriteLine("Index already exists for version" + version);
return(ret);
}
public IndexModel GetIndexInfo(int id = 1)
{
IndexEntity entity = _index.Find(id);
LogHelper.WriteLog($"HomeService 获取到Index了：{entity.ToJson()}");
IndexModel model = Mapper.Map <IndexModel>(entity);
List <VideoEntity> startVideo = _video.GetStartVideo().OrderByDescending(v => v.UpdateTime).Take(2).ToList();
LogHelper.WriteLog($"HomeService 获取到Video了：{startVideo.ToJson()}");
if (startVideo.Count > 0)
{
model.RecommendOne = Mapper.Map <StartRecommendModel>(startVideo.FirstOrDefault());
}
if (startVideo.Count > 1)
{
model.RecommendTwo = Mapper.Map <StartRecommendModel>(startVideo.LastOrDefault());
}
return(model);
}
public string RetrievePageTitle(string url)
{
string title = "index not found :(";
TableQuery <IndexEntity> query = new TableQuery <IndexEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, IndexEntity.Base64Encode(url)));
IndexEntity index = null;
foreach (IndexEntity entity in indexTable.ExecuteQuery(query))
{
index = entity;
break;
}
if (index != null)
{
title = index.Title;
}
return(new JavaScriptSerializer().Serialize(title));
}
public void Update(IndexModel model)
{
IndexEntity index = _index.Find(model.Id);
Mapper.Map(model, index);
}
public int InSert(IndexModel model)
{
IndexEntity index = Mapper.Map <IndexEntity>(model);
return(_index.Insert(index));
}
public async Task<long> createIndexIfNotExist(string version)
{
long ret = await getIndex(version);
if (ret == -1)
{
IndexEntity i = new IndexEntity(version);
i.index = 0;
TableOperation insertOperation = TableOperation.Insert(i);
await _indextable.ExecuteAsync(insertOperation);
Console.WriteLine("Created index for version " + version);
return 0;
}
Console.WriteLine("Index already exists for version" + version);
return ret;
}
