protected override List <EntityCase> HandleCore(IndexEntity message) { var department = _db.EntityCase .Where(d => (message.ID == null) || d.Id == message.ID).ToList <EntityCase>(); return(department); }
public string GetRecentUrlsCrawled(int n) { List <string> results = new List <string>(); TableQuery <IndexEntity> query = new TableQuery <IndexEntity>(); int i = 0; foreach (IndexEntity entity in indexTable.ExecuteQuery(query)) { if (i == n) { break; } i++; results.Add(IndexEntity.Base64Decode(entity.PartitionKey)); } if (results.Count == 0) { results.Add("Index is empty"); } return(new JavaScriptSerializer().Serialize(results)); }
private async Task createIndexIfNotExist() { long ret = await getIndex(); if (ret == -1) { IndexEntity i = new IndexEntity(_version); i.index = 0; TableOperation insertOperation = TableOperation.Insert(i); await _indextable.ExecuteAsync(insertOperation); } }
public void Crawl(string url) { if (!visitedUrls.ContainsKey(url)) { HtmlDocument document; try { document = web.Load(url); } catch { Logger.Instance.Log(Logger.LOG_ERROR, "html dom parsing failed in WebCrawler.Crawl() for " + url); return; } Link parentLink = new Link(url); HtmlNode[] nodes = document.DocumentNode.SelectNodes("//title").ToArray(); string title = "Page indexed, but no <title> tag found"; foreach (HtmlNode item in nodes) { title = item.InnerHtml; break; // there should only be one title, if there are more then pick the 1st one arbitrarily } IndexEntity indexEntity = new IndexEntity(url, title); // also add page date TableOperation insertOperation = TableOperation.Insert(indexEntity); urlTable.Execute(insertOperation); visitedUrls.Add(url, true); nodes = document.DocumentNode.SelectNodes("//a").ToArray(); foreach (HtmlNode item in nodes) { string linkPath = item.GetAttributeValue("href", ""); // could be relative or absolute or external string link = parentLink.buildUrl(linkPath); if (!visitedUrls.ContainsKey(link) && urlValidator.IsUrlValidCrawling(link)) { visitedUrls.Add(link, true); UrlMessage urlEntity = new UrlMessage(UrlMessage.URL_TYPE_HTML, link); // Add message CloudQueueMessage message = new CloudQueueMessage(urlEntity.ToString()); urlQueue.AddMessage(message); } } } }
public async Task <bool> Init() { var table = await TableExecutor <TableEntity> .CreateTableAsync("shorturls", _cloudStorage); for (int i = 0; i < 99; i++) { var partkey = i.ToString("00"); var index = await TableExecutor <IndexEntity> .PointQueryAsync(table, partkey, "index"); if (index == null) { index = new IndexEntity(partkey, "index", 0L); await TableExecutor <IndexEntity> .InsertOrMergeEntityAsync(table, index); } TableIndex.AddOrUpdate(partkey, index.Index, (k, v) => v = index.Index); } return(true); }
public async Task <long> createIndexIfNotExist(string version) { long ret = await getIndex(version); if (ret == -1) { IndexEntity i = new IndexEntity(version); i.index = 0; TableOperation insertOperation = TableOperation.Insert(i); await _indextable.ExecuteAsync(insertOperation); Console.WriteLine("Created index for version " + version); return(0); } Console.WriteLine("Index already exists for version" + version); return(ret); }
public IndexModel GetIndexInfo(int id = 1) { IndexEntity entity = _index.Find(id); LogHelper.WriteLog($"HomeService 获取到Index了:{entity.ToJson()}"); IndexModel model = Mapper.Map <IndexModel>(entity); List <VideoEntity> startVideo = _video.GetStartVideo().OrderByDescending(v => v.UpdateTime).Take(2).ToList(); LogHelper.WriteLog($"HomeService 获取到Video了:{startVideo.ToJson()}"); if (startVideo.Count > 0) { model.RecommendOne = Mapper.Map <StartRecommendModel>(startVideo.FirstOrDefault()); } if (startVideo.Count > 1) { model.RecommendTwo = Mapper.Map <StartRecommendModel>(startVideo.LastOrDefault()); } return(model); }
public string RetrievePageTitle(string url) { string title = "index not found :("; TableQuery <IndexEntity> query = new TableQuery <IndexEntity>() .Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, IndexEntity.Base64Encode(url))); IndexEntity index = null; foreach (IndexEntity entity in indexTable.ExecuteQuery(query)) { index = entity; break; } if (index != null) { title = index.Title; } return(new JavaScriptSerializer().Serialize(title)); }
public void Update(IndexModel model) { IndexEntity index = _index.Find(model.Id); Mapper.Map(model, index); }
public int InSert(IndexModel model) { IndexEntity index = Mapper.Map <IndexEntity>(model); return(_index.Insert(index)); }
public async Task<long> createIndexIfNotExist(string version) { long ret = await getIndex(version); if (ret == -1) { IndexEntity i = new IndexEntity(version); i.index = 0; TableOperation insertOperation = TableOperation.Insert(i); await _indextable.ExecuteAsync(insertOperation); Console.WriteLine("Created index for version " + version); return 0; } Console.WriteLine("Index already exists for version" + version); return ret; }