private static void BjCrawler()
{
TaskEntity model = new TaskEntity
{
OperatorName = "张雪艳",
Unique = Guid.NewGuid(),
CreateTime = DateTime.Now,
TaskName = "测试任务",
TaskStateDicId = 1,
};
//Proxy.Proxy.GetInstance().ProxySave(100, 10);
//Proxy.Proxy.GetInstance().ValidateCanUse(7);
var OperatorName = "张雪艳"; //操作人
//待查数据初始化
var lists = File.ReadAllLines("E:/1.txt", Encoding.Default).ToList();
new BaseData(model).InsertMetadata(lists, OperatorName, model, taskEntity =>
{
//抓取数据
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
bjqyxy.CrawlerWork();
});
}
public ActionResult Create(TaskEntity model, HttpPostedFileBase txtfile)
{
var stream = txtfile.InputStream;
var streamread = new StreamReader(stream, Encoding.Default);
var companyList = new List <string>();
while (!streamread.EndOfStream)
{
companyList.Add(streamread.ReadLine());
}
model.TaskType = EnumTaskType.BjCrawler;
model.TaskStateDicId = 1;
model.TaskNum = companyList.Count;
model.CreateTime = DateTime.Now;
model.IsSingelSearch = false;
new TaskDomain().Add(model);
new Task(() =>
{
new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
{
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
new Task(() => { bjqyxy.CrawlerWork(); }).Start();
});
}).Start();
return(Json(new { msg = $"成功上传了任务文件，系统接受到{companyList.Count}条记录，正在导入系统中。。。" }));
}
public ActionResult GoGather(TaskEntity model)
{
//Thread.Sleep(1000 * 3); //默认等待三秒
//抓取数据
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(model, t => t.TaskGuid.Equals(model.Unique));
new Task(() => { bjqyxy.CrawlerWork(); }).Start();
return(Json(new { state = "nothion" }));
}
public ActionResult SingelSearch(string guid = null, string searchInfo = null)
{
CrawlerEntity crawlerEntity = null;
if (string.IsNullOrWhiteSpace(guid) && string.IsNullOrWhiteSpace(searchInfo))
{
ViewBag.Guid = Guid.NewGuid();
}
else
{
ViewBag.Guid = guid;
var count = new TargeCompanyDomain().Get(t => t.CompanyName.Equals(searchInfo))?.Count;
if (count > 0)
{
//历史记录中已存在
}
else
{
//上网检索
List <string> companyList = new List <string> {
searchInfo
};
TaskEntity model = new TaskEntity();
model.TaskType = EnumTaskType.BjCrawler;
model.TaskName = $"单个任务[{DateTime.Now.ToString("G")}]";
model.Unique = Conv.ToGuid(guid);
model.TaskStateDicId = 1;
model.TaskNum = 1;
model.CreateTime = DateTime.Now;
model.IsSingelSearch = true;
new TaskDomain().Add(model);
new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
{
Task[] tasks = new Task[4];
for (int i = 0; i < 4; i++)
{
tasks[i] = new Task(() =>
{
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
bjqyxy.SingelSearch(searchInfo);
});
tasks[i].Start();
}
Task.WaitAny(tasks);
});
}
crawlerEntity = new CrawlerDomain().Get(t => t.搜索名称 == searchInfo && t.称 != null).FirstOrDefault();
}
return(View(crawlerEntity));
}
public ActionResult SingelSearch(string guid = null, string searchInfo = null)
{
CrawlerEntity crawlerEntity = null;
if (string.IsNullOrWhiteSpace(guid) && string.IsNullOrWhiteSpace(searchInfo))
{
ViewBag.Guid = Guid.NewGuid();
}
else
{
ViewBag.Guid = guid;
var count = new TargeCompanyDomain().Get(t => t.CompanyName.Equals(searchInfo))?.Count;
if (count > 0)
{
//历史记录中已存在
}
else
{
//上网检索
List<string> companyList = new List<string> { searchInfo };
TaskEntity model = new TaskEntity();
model.TaskType = EnumTaskType.BjCrawler;
model.TaskName = $"单个任务[{DateTime.Now.ToString("G")}]";
model.Unique = Conv.ToGuid(guid);
model.TaskStateDicId = 1;
model.TaskNum = 1;
model.CreateTime = DateTime.Now;
model.IsSingelSearch = true;
new TaskDomain().Add(model);
new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
{
Task[] tasks = new Task[4];
for (int i = 0; i < 4; i++)
{
tasks[i] = new Task(() =>
{
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
bjqyxy.SingelSearch(searchInfo);
});
tasks[i].Start();
}
Task.WaitAny(tasks);
});
}
crawlerEntity = new CrawlerDomain().Get(t => t.搜索名称 == searchInfo && t.名称 != null).FirstOrDefault();
}
return View(crawlerEntity);
}
private static void BjCrawler()
{
TaskEntity model = new TaskEntity
{
OperatorName = "张雪艳",
Unique = Guid.NewGuid(),
CreateTime = DateTime.Now,
TaskName = "测试任务",
TaskStateDicId = 1,
};
//Proxy.Proxy.GetInstance().ProxySave(100, 10);
//Proxy.Proxy.GetInstance().ValidateCanUse(7);
var OperatorName = "张雪艳"; //操作人
//待查数据初始化
var lists = File.ReadAllLines("E:/1.txt", Encoding.Default).ToList();
new BaseData(model).InsertMetadata(lists, OperatorName, model, taskEntity =>
{
//抓取数据
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
bjqyxy.CrawlerWork();
});
}
public ActionResult Create(TaskEntity model, HttpPostedFileBase txtfile)
{
var stream = txtfile.InputStream;
var streamread = new StreamReader(stream, Encoding.Default);
var companyList = new List<string>();
while (!streamread.EndOfStream)
companyList.Add(streamread.ReadLine());
model.TaskType = EnumTaskType.BjCrawler;
model.TaskStateDicId = 1;
model.TaskNum = companyList.Count;
model.CreateTime = DateTime.Now;
model.IsSingelSearch = false;
new TaskDomain().Add(model);
new Task(() =>
{
new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
{
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
new Task(() => { bjqyxy.CrawlerWork(); }).Start();
});
}).Start();
return Json(new { msg = $"成功上传了任务文件，系统接受到{companyList.Count}条记录，正在导入系统中。。。" });
}
public ActionResult GoGather(TaskEntity model)
{
//Thread.Sleep(1000 * 3); //默认等待三秒
//抓取数据
var bjqyxy = new Crawler.Bjqyxy.BjCrawler(model, t => t.TaskGuid.Equals(model.Unique));
new Task(() => { bjqyxy.CrawlerWork(); }).Start();
return Json(new { state = "nothion" });
}