public async Task <bool> IsUrlVisited(string url)
{
var urlMD5 = NScrapyHelper.GetMD5FromString(url);
var connection = RedisSchedulerContext.Current.Connection;
var urlSetName = $"{RedisSchedulerContext.Current.ReceiverQueue}.VisitedURLMD5";
if (await connection.GetDatabase().SetContainsAsync(urlSetName, urlMD5))
{
return(true);
}
await connection.GetDatabase().SetAddAsync(urlSetName, urlMD5);
return(false);
}
private void ListenToRedisTopic()
{
var connection = RedisSchedulerContext.Current.Connection;
while (true)
{
var lockToken = Guid.NewGuid();
RedisResponseMessage responseMessage = null;
RedisValue value = new RedisValue();
try
{
RedisSchedulerContext.Current.GetLock($"{RedisSchedulerContext.Current.ResponseQueue}.Lock", lockToken.ToString());
if (RedisSchedulerContext.Current.Connection.GetDatabase().ListLength(RedisSchedulerContext.Current.ResponseQueue) > 0)
{
value = RedisSchedulerContext.Current.Connection.GetDatabase().ListRightPop(RedisSchedulerContext.Current.ResponseQueue);
}
}
catch (Exception ex)
{
NScrapyContext.CurrentContext.Log.Error($"Error getting value from Redis", ex);
}
finally
{
RedisSchedulerContext.Current.ReleaseLock($"{RedisSchedulerContext.Current.ResponseQueue}.Lock", lockToken.ToString());
}
if (!value.HasValue)
{
continue;
}
responseMessage = JsonConvert.DeserializeObject <RedisResponseMessage>(value);
if (registedCallback.ContainsKey(responseMessage.CallbacksFingerprint))
{
var callback = registedCallback[responseMessage.CallbacksFingerprint];
var response = NScrapyHelper.DecompressResponse(responseMessage.Payload);
var task = new Task(() => callback(response));
task.ContinueWith(t => NScrapyContext.CurrentContext.Log.Error($"Error Process Callback {responseMessage.CallbacksFingerprint}", task.Exception), TaskContinuationOptions.OnlyOnFaulted);
task.Start();
}
}
}
private static void ProcessRequestAndSendBack(string responseQueueName, string requestMessage)
{
var requestObj = JsonConvert.DeserializeObject <RedisRequestMessage>(requestMessage);
var request = new HttpRequest()
{
URL = requestObj.URL
};
var result = Downloader.Downloader.SendRequestAsync(request);
result.ContinueWith(async u =>
{
var resultPayload = string.Empty;
try
{
resultPayload = JsonConvert.SerializeObject(u.Result);
}
catch (Exception ex)
{
DownloaderContext.Context.Log.Error($"Serialize response for {request.URL} failed!", ex);
}
resultPayload = await NScrapyHelper.CompressString(resultPayload);
var responseMessage = new RedisResponseMessage()
{
URL = requestObj.URL,
CallbacksFingerprint = requestObj.CallbacksFingerprint,
Payload = resultPayload
};
//var publishResult = RedisManager.Connection.GetSubscriber().Publish(responseQueueName, $"{JsonConvert.SerializeObject(responseMessage)}");
var publishResult = RedisManager.Connection.GetDatabase().ListLeftPush(responseQueueName, $"{JsonConvert.SerializeObject(responseMessage)}");
DownloaderContext.Context.Log.Info($"Sending request to {request.URL} success!");
},
TaskContinuationOptions.OnlyOnRanToCompletion);
result.ContinueWith(u =>
DownloaderContext.Context.Log.Info($"Sending request to {request.URL} failed", result.Exception.InnerException),
TaskContinuationOptions.OnlyOnFaulted);
}
