public async Task RenderAsync(ISpiderContainer container) { var uri = string.IsNullOrWhiteSpace(name) ? container.Data.ToString() : container.GetAttribute(name); if (string.IsNullOrWhiteSpace(uri) || uri.Length > 500) { await container.NextAsync(); return; } var fromUri = new Uri(container.Url.Source); var toUri = new Uri(fromUri, uri); var fullUri = toUri.ToString(); container.Application.UrlProvider.Add(fullUri); var item = container.Application.UrlProvider.Get(fullUri); if (item == null) { await container.NextAsync(); return; } await LoadNext(container, item); }
public async Task RenderAsync(ISpiderContainer container) { var logger = container.Logger; var segmenter = new JiebaNet.Analyser.TfidfExtractor(); var tags = segmenter.ExtractTags(container.Data.ToString()); logger?.Debug(string.Join(" ", tags)); var client = new Client(); if (!string.IsNullOrEmpty(apiToken)) { client.Headers.Add("Authorization", "Bearer " + apiToken); } var data = new PostForm(); data.Title = container.GetAttribute("title"); data.Description = container.GetAttribute("description"); data.Content = container.GetAttribute("content"); data.Link = container.GetAttribute("url"); data.Keywords = tags; await client.PostAsync(apiUri, JsonConvert.SerializeObject(data), "application/json"); await container.NextAsync(); }