/// <summary>
/// 开始爬取数据
/// </summary>
/// <param name="interval">单位:毫秒</param>
public void StartCrawler(WebProxy webProxy, int interval, IEnumerable <string> stockCodes)
{
if (_isStarted)
{
return;
}
_isStarted = true;
_isStoped = false;
_interval = interval;
_webProxy = webProxy;
_configModel = ConfigJsonHelper.GetConfigModel();
CheckConfig();
_pageSize = GetPageSize();
_taskWorkCodeCount = _configModel.ClawlerConfigData.CodeCountPerThread;
_monitorStocks = new List <string>(stockCodes);
//分配一个线程专门从缓存中取出数据及数据转换并通过事件通知订阅者
Task.Factory.StartNew(() =>
{
while (!_isStoped)
{
try
{
Tuple <string, string> tuple;
var dequeueSuccesful = _responseJsonQueue.TryDequeue(out tuple);
if (dequeueSuccesful)
{
var queryStockTransactionResponseModel = JsonHelper.DeserializeJsonToObject <QueryStockTransactionResponseModel>(tuple.Item2);
var addedStr = new List <string>();
if (_datasDic.ContainsKey(tuple.Item1))
{
addedStr = queryStockTransactionResponseModel.Value.Datas.Except(_datasDic[tuple.Item1]).ToList();
_datasDic[tuple.Item1].AddRange(addedStr);
}
else
{
addedStr = new List <string>(queryStockTransactionResponseModel.Value.Datas);
_datasDic.Add(tuple.Item1, addedStr);
}
if (addedStr.Any())
{
List <StockTransactionModel> addedModels = DtoHelper.GetStockTransactionModels(addedStr);
addedModels.ForEach(x => x.Code = tuple.Item1);
StockRealTimeDealUpdatedEven?.Invoke(new StockRealTimeDealUpdatedEventArgs(addedModels));
}
}
}
catch (Exception)
{
}
Thread.Sleep(1);
}
}, TaskCreationOptions.LongRunning);
var count = _monitorStocks.Count % _taskWorkCodeCount == 0 ? _monitorStocks.Count / _taskWorkCodeCount : _monitorStocks.Count / _taskWorkCodeCount + 1;
for (int i = 1; i <= count; i++)
{
var codes = _monitorStocks.Take(i * _taskWorkCodeCount).Skip((i - 1) * _taskWorkCodeCount);
WorkTaskModel model = new WorkTaskModel();
model.StockCodes = new List <string>(codes);
model.Cts = new CancellationTokenSource();
//System.Diagnostics.Process.GetCurrentProcess().ProcessorAffinity = (IntPtr)0X007F;
//指定线程在哪个cpu上运行,由于此程序的cpu占用率过高,想通过这种方式降低cpu
//此方式确实可以降低cpu,但是数据处理速度慢了很多,所以还是放弃这种方式
int userCount = 4;
int core = i % userCount == 0 ? 1 + i % userCount : i % userCount;
model.WorkTask = new Task(() => { DoCrawler(model.StockCodes, model.Cts.Token, core); }, model.Cts.Token, TaskCreationOptions.LongRunning);
_workTasks.Add(model);
}
_workTasks.ForEach(work =>
{
work.WorkTask.Start();
});
}
