private async Task RegisterAgentAsync(string topic, CancellationToken stoppingToken)
{
var consumer = new AsyncMessageConsumer <byte[]>(topic);
consumer.Received += HandleMessageAsync;
await _messageQueue.ConsumeAsync(consumer, stoppingToken);
_consumers.Add(consumer);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
_logger.LogInformation("Agent center service is starting");
await _agentStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(TopicNames.AgentCenter);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
_logger.LogWarning("Received empty message");
return;
}
if (message is Register register)
{
if (_distributed)
{
_logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
}
await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
register.ProcessorCount,
register.TotalMemory));
}
else if (message is Heartbeat heartbeat)
{
if (_distributed)
{
_logger.LogInformation($"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
}
await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
heartbeat.FreeMemory, heartbeat.CpuLoad));
}
else
{
var msg = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
_logger.LogWarning($"Not supported message: {msg}");
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Agent center service started");
}
catch (Exception e)
{
_logger.LogCritical(e.ToString());
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
_logger.LogInformation("Agent center service is starting");
await _agentStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(Topics.AgentCenter);
_consumer.Received += async bytes =>
{
object message;
try
{
message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
return;
}
}
catch (Exception e)
{
_logger.LogError($"Deserialize message failed: {e}");
return;
}
switch (message)
{
case Messages.Agent.Register register:
{
if (_distributed)
{
_logger.LogInformation($"Register agent: {register.AgentId}, {register.AgentName}");
}
await _agentStore.RegisterAsync(new AgentInfo(register.AgentId, register.AgentName,
register.ProcessorCount,
register.Memory));
break;
}
case Messages.Agent.Heartbeat heartbeat:
{
if (_distributed)
{
_logger.LogInformation(
$"Receive heartbeat: {heartbeat.AgentId}, {heartbeat.AgentName}");
}
await _agentStore.HeartbeatAsync(new AgentHeartbeat(heartbeat.AgentId, heartbeat.AgentName,
heartbeat.AvailableMemory, heartbeat.CpuLoad));
break;
}
default:
{
var msg = JsonSerializer.Serialize(message);
_logger.LogWarning($"Message not supported: {msg}");
break;
}
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Agent center service started");
}
catch (Exception e)
{
_logger.LogCritical(e.ToString());
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Statistics service starting");
await _statisticsStore.EnsureDatabaseAndTableCreatedAsync();
_consumer = new AsyncMessageConsumer <byte[]>(Topics.Statistics);
_consumer.Received += async bytes =>
{
var message = await bytes.DeserializeAsync(stoppingToken);
if (message == null)
{
_logger.LogWarning("Received empty message");
return;
}
if (message is Messages.Statistics.Success success)
{
await _statisticsStore.IncreaseSuccessAsync(success.SpiderId);
}
else if (message is Messages.Statistics.Start start)
{
await _statisticsStore.StartAsync(start.SpiderId, start.SpiderName);
}
else if (message is Messages.Statistics.Failure failure)
{
await _statisticsStore.IncreaseFailureAsync(failure.SpiderId);
}
else if (message is Messages.Statistics.Total total)
{
await _statisticsStore.IncreaseTotalAsync(total.SpiderId, total.Count);
}
else if (message is Messages.Statistics.Exit exit)
{
await _statisticsStore.ExitAsync(exit.SpiderId);
}
else if (message is Messages.Statistics.RegisterAgent registerAgent)
{
await _statisticsStore.RegisterAgentAsync(registerAgent.AgentId, registerAgent.AgentName);
}
else if (message is Messages.Statistics.AgentSuccess agentSuccess)
{
await _statisticsStore.IncreaseAgentSuccessAsync(agentSuccess.AgentId,
agentSuccess.ElapsedMilliseconds);
}
else if (message is Messages.Statistics.AgentFailure agentFailure)
{
await _statisticsStore.IncreaseAgentFailureAsync(agentFailure.AgentId,
agentFailure.ElapsedMilliseconds);
}
else if (message is Messages.Statistics.Print print)
{
var statistics = await _statisticsStore.GetSpiderStatisticsAsync(print.SpiderId);
if (statistics != null)
{
var left = statistics.Total >= statistics.Success
? (statistics.Total - statistics.Success - statistics.Failure).ToString()
: "unknown";
var now = DateTimeOffset.Now;
var speed = (decimal)(statistics.Success /
(now - (statistics.Start ?? now.AddMinutes(-1))).TotalSeconds);
_logger.LogInformation(
$"{print.SpiderId} total {statistics.Total}, speed: {decimal.Round(speed, 2)}, success {statistics.Success}, failure {statistics.Failure}, left {left}");
}
}
else
{
var log = Encoding.UTF8.GetString(JsonSerializer.SerializeToUtf8Bytes(message));
_logger.LogWarning($"Not supported message: {log}");
}
};
await _messageQueue.ConsumeAsync(_consumer, stoppingToken);
_logger.LogInformation("Statistics service started");
}
