public async Task <BlockOutput> Pop(NodeHeartbeatInput heartbeat)
{
using (var conn = CreateDbConnection())
{
if (conn.State == ConnectionState.Closed)
{
conn.Open();
}
var transaction = conn.BeginTransaction();
BlockOutput output = new BlockOutput {
Command = Command.Download
};
try
{
var running = await _runningService.Pop(conn, transaction, heartbeat.Runnings);
output.Identity = running.Identity;
var block = await conn.QueryFirstOrDefaultAsync <Block>($@"SELECT TOP 1 * FROM block WHERE
{LeftEscapeSql}identity{RightEscapeSql}=@Identity WHERE status = 1 or status = 5", new { running.Identity }, transaction);
if (block != null)
{
await conn.ExecuteAsync($"UPDATE block SET status =@Status, lastmodificationtime={GetDateSql} WHERE blockid=@BlockId",
new { Status = BlockStatus.Using, block.BlockId }, transaction);
var requests = await _requestQueueService.GetByBlockId(block.BlockId);
output.Requests = requests.Select(r =>
{
var request = JsonConvert.DeserializeObject <Request>(r.Request);
return(new RequestOutput {
Content = request.Content, CycleTriedTimes = request.CycleTriedTimes, Depth = request.Depth, Method = request.Method, Origin = request.Origin, Referer = request.Referer, Url = request.Url
});
}).ToList();
output.BlockId = block.BlockId;
output.Site = JsonConvert.DeserializeObject <Site>(running.Site);
output.ThreadNum = running.ThreadNum;
}
transaction.Commit();
}
catch
{
try
{
transaction.Rollback();
}
catch (Exception ex)
{
_logger.LogError(ex.ToString());
}
throw;
}
return(output);
}
}
private void HandleBlock(BlockOutput block)
{
_runnings.Add(block.Identity);
BlockInput blockinput = new BlockInput {
Id = block.Id, Identity = block.Identity, Results = new List <RequestInput>()
};
try
{
if (block.Requests == null || block.Requests.Count == 0)
{
return;
}
Parallel.ForEach(block.Requests, new ParallelOptions {
MaxDegreeOfParallelism = block.ThreadNum
}, (dto) =>
{
for (int i = 0; i < _options.RetryDownload; ++i)
{
var requestId = dto.Identity;
try
{
var response = _downloader.Download(dto.ToRequest());
lock (blockinput)
{
blockinput.Results.Add(new RequestInput {
CycleTriedTimes = i, Content = response.Content, Identity = requestId, ResponseTime = DateTime.Now, StatusCode = (int)response.StatusCode, TargetUrl = response.TargetUrl
});
}
Log.Logger.Information($"Request {requestId} success.");
break;
}
catch (Exception e)
{
Log.Logger.Error($"Request {requestId} failed: {e}");
Thread.Sleep(300);
}
}
Thread.Sleep(block.Site.SleepTime);
});
}
catch (Exception e)
{
blockinput.Exception = e.ToString();
}
finally
{
_runnings.Remove(block.Identity);
var json = JsonConvert.SerializeObject(blockinput);
for (int j = 0; j < _options.RertyPush; ++j)
{
try
{
var request = new Request(_pushBlockinputUrl);
request.Method = HttpMethod.Post;
request.Site = _brokerSite;
request.Content = json;
request.CompressMode = CompressMode.Lz4;
_downloader.Download(request);
Log.Logger.Information($"Push blockinput {block.Id}, identity {block.Identity} success.");
break;
}
catch (Exception e)
{
Log.Logger.Error($"Push blockinput failed: {e}.");
}
}
}
}
