public Resultado Excluir(Item oItem)
{
Resultado resultado = new Resultado();
resultado = new ItemProcess().Atualiza(oItem);
return(resultado);
}
public Resultado Inserir(Item oItem)
{
Resultado resultado = new Resultado();
resultado = new ItemProcess().Incluir(oItem);
return(resultado);
}
public async Task <IActionResult> Get()
{
HttpStep httpStep = HttpStep.CreateGetStep("label", null, "https://google.com");
ItemProcess itemProcess = new ItemProcess("desc", "label", 1, new List <Step>()
{
httpStep
});
await itemProcessRepository.AddAsync(itemProcess);
await itemProcessRepository.SaveAsync();
return(Ok());
}
private async Task RunProcessAsync(ItemProcess itemProcess)
{
var itemsIEnum = await itemRepository.GetNonStartedItemsByProcessIdAsync(itemProcess.Id);
var items = itemsIEnum.ToList();
Task[] tasks = new Task[items.Count];
for (int i = 0; i < items.Count; i++)
{
tasks[i] = processService.ProcessPendingItemAsync(itemProcess, items[i], new CancellationToken());
}
await Task.WhenAll(tasks);
}
ProcessViewModel(ItemProcess data)
{
Id = data.Id;
ShowDivOne = data.DivOneVisible.HasValue ? data.DivOneVisible.Value : false;
ShowDivTwo = data.DivTwoVisible.HasValue ? data.DivTwoVisible.Value : false;
}
private static void DoSimpleCrawler()
{
//EntityManager.PrepareWithAutoRefreash();
//1.通过TaskProvider获取Simple Crawler任务
TaskProvider taskProvider = new TaskProvider();
string lockCrawler = DES.Encrypt("NC31ASTORM", DataAccess.PalasKey);
Palas.Protocol.PFTaskInfo taskInfo = taskProvider.GetTask(lockCrawler, ConfigurationManager.AppSettings["CrawlConductUrl"] + "Task/GetCrawlTask");
Console.WriteLine(string.Format("GetTask: {0}", taskInfo.TaskId));
//2.使用TaskResolve将任务进行分解
TaskResolve taskResolve = new TaskResolve();
List <Palas.Protocol.PFCrawlInfo> crawlInfos = taskResolve.ResolveTask(taskInfo);
foreach (var crawlInfo in crawlInfos)
{
//3.使用HttpRequest获取网页信息
HttpRequest httpRequest = new HttpRequest();
Palas.Protocol.PFCrawlResponse crawlResponse = httpRequest.DoWebRequest(crawlInfo);
//4.使用ParserHtml解析HTML信息
Palas.Framework.Module.ParserHtml parserHtml = new Palas.Framework.Module.ParserHtml();
List <string> papers = parserHtml.Parser(crawlResponse, taskInfo);
Console.WriteLine(string.Format("Crawl paper count: {0}", papers.Count));
foreach (string paper in papers)
{
//5.使用ItemProcess对文章信息进行预处理
ItemProcess itemProcess = new ItemProcess();
string item = itemProcess.Process(paper, taskInfo.BaseInfo);
if (!string.IsNullOrEmpty(item))
{
Palas.Protocol.PFItemToAnalyze pfItemToAnalyze = new Palas.Protocol.PFItemToAnalyze();
pfItemToAnalyze.Item = JsonConvert.DeserializeObject <Palas.Protocol.PFItem>(item);
pfItemToAnalyze.CrawlRecode = taskInfo.BaseInfo;
pfItemToAnalyze.AnalyzeFlag = taskInfo.BaseInfo.AnalyzeFlag;
Palas.Protocol.PFItemToAnalyze analyzeResult = AnalyzeItem.Analyzer(pfItemToAnalyze, Palas.Protocol.PFAnalyzeFlag.Splite);
//6.使用AnalyzeFirst对文章进行第一次分析
MultriAnalyzeFlag analyzeFlags = AnalyzeItem.BuildAnalyzeFlag((Enums.AnalyzeFlag)analyzeResult.AnalyzeFlag);
Palas.Protocol.PFItemToAnalyze analyzeFirstResult = AnalyzeItem.AnalyzerFirst(analyzeResult, analyzeFlags);
//7.使用AnalyzeSecond对文章进行第二次分析
Palas.Protocol.PFItemToAnalyze analyzeSecondResult = AnalyzeItem.AnalyzeSecond(analyzeFirstResult, analyzeFlags);
//8.使用AnalyzeIssue对文章进行分Issue分析
Palas.Protocol.PFItemToAnalyze analyzeIssueResult = AnalyzeItem.IssueAnalyzer(analyzeSecondResult, analyzeFlags);
//9.使用IndexThenDup将文章去重索引到ES
if (analyzeIssueResult != null)
{
//此处对Item进行一次转换
Palas.Protocol.PFItem pfItem = analyzeIssueResult.Item;
Item _item = TypeExchangeUtility.ExchangeItem(pfItem);
_item = FilterIssue.FilterExcludeExpression(_item);
Enums.ProcessStatus result = Enums.ProcessStatus.Failed;
int retry = 0;
do
{
try
{
//retry++;
ESAccess.IndexOnly(_item);
result = Enums.ProcessStatus.Succeeded;
//result = DupThenIndexBusiness.DupThenIndexItem(_item);
//if (result == Enums.ProcessStatus.Failed)
//Thread.Sleep(new TimeSpan(0, 0, 30));
}
catch //(Exception ex)
{
result = Enums.ProcessStatus.Failed;
Thread.Sleep(new TimeSpan(0, 1, 00));
}
}while (result == Enums.ProcessStatus.Failed && retry < 3);
//10.判断是否成功
//Assert.AreNotEqual(Enums.ProcessStatus.Failed, result);
Console.WriteLine(string.Format("Index paper: {0}, Status: {1}", _item.ItemID, result.ToString()));
}
}
}
}
}
