//public void Clear()
//{
// var redisScheduler = spider.Scheduler as Scheduler.RedisScheduler;
// if (redisScheduler != null)
// {
// redisScheduler.Clear(spider);
// }
//}
private void DoValidate()
{
if (SpiderContext.Validations == null)
{
return;
}
RedisServer redis = new RedisServer(ConfigurationManager.Get("redisHost"), 6379, ConfigurationManager.Get("redisPassword"));
string key = "locker-validate-" + Name;
try
{
var validations = SpiderContext.Validations.GetValidations();
if (validations != null && validations.Count > 0)
{
foreach (var validation in validations)
{
validation.CheckArguments();
}
}
if (redis != null)
{
while (!redis.LockTake(key, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
}
var lockerValue = redis?.HashGet(ValidateStatusName, Name);
bool needInitStartRequest = lockerValue != "validate finished";
if (needInitStartRequest)
{
Logger.Info("开始数据验证 ...");
if (validations != null && validations.Count > 0)
{
MailBodyBuilder builder = new MailBodyBuilder(Name, SpiderContext.Validations.Corporation);
foreach (var validation in validations)
{
builder.AddValidateResult(validation.Validate());
}
string mailBody = builder.Build();
using (EmailClient client = new EmailClient(SpiderContext.Validations.EmailSmtpServer, SpiderContext.Validations.EmailUser, SpiderContext.Validations.EmailPassword, SpiderContext.Validations.EmailSmtpPort))
{
client.SendMail(new EmaillMessage($"{Name} " + "validation report", mailBody, SpiderContext.Validations.EmailTo) { IsHtml = true });
}
}
}
else
{
Logger.Info("有其他线程执行了数据验证.");
}
if (needInitStartRequest)
{
redis?.HashSet(ValidateStatusName, Name, "validate finished");
}
}
catch (Exception e)
{
Logger.Error(e.Message, e);
}
finally
{
redis?.LockRelease(key, 0);
}
}
public void RedisBaseTest()
{
RedisServer r = new RedisServer("localhost");
r.Db = 3;
#region set
long i;
r.SetLength("a");
r.SetAdd("foo", "bar");
r.FlushDb();
Assert.IsTrue((i = r.SetLength("foo")) == 0, "there should be no keys but there were {0}", i);
r.SetAdd("foo", "bar");
Assert.IsTrue((i = r.SetLength("foo")) == 1, "there should be one key but there were {0}", i);
r.SetAdd("foo bär", "bär foo");
r.SetAdd("foo", "bär foo");
Assert.IsTrue((i = r.SetLength("foo")) == 2, "there should be two keys but there were {0}", i);
Assert.IsTrue(r.TypeOf("foo") == KeyType.Set, "type is not string");
r.SetAdd("bar", "foo");
Assert.IsTrue(r.SetContains("bar", "foo"));
var mems = r.SetMembers("foo");
Assert.AreEqual("bar", mems[0]);
Assert.AreEqual("bär foo", mems[1]);
r.SetRemove("foo", "bar");
mems = r.SetMembers("foo");
Assert.AreEqual("bär foo", mems[0]);
Assert.AreEqual(1, mems.Count);
string item = r.SetPop("foo");
Assert.AreEqual("bär foo", item);
Assert.AreEqual(0, r.SetLength("foo"));
#endregion
#region hasset
r.HashSet("set", "a", "a");
r.HashSet("set", "b", "b");
r.HashSet("set", "c", "c");
var hash = r.HashGetAll("set");
Assert.AreEqual(hash.Length, 3);
r.HashDelete("set", "a");
hash = r.HashGetAll("set");
Assert.AreEqual(hash.Length, 2);
Assert.AreEqual(true, r.HashExists("set", "b"));
Assert.AreEqual("b", r.HashGet("set", "b"));
Assert.AreEqual(2, r.HashLength("set"));
#endregion
#region storedset
r.SortedSetAdd("sortedset", "a", 0L);
r.SortedSetAdd("sortedset", "b", 0L);
r.SortedSetAdd("sortedset", "c", 0L);
r.SortedSetAdd("sortedset", "d", 0L);
Assert.AreEqual(4, r.SortedSetLength("sortedset"));
r.SortedSetRemove("sortedset", "a");
Assert.AreEqual(3, r.SortedSetLength("sortedset"));
string[] sr = r.SortedSetRangeByRank("sortedset", 0, 2);
Assert.AreEqual("b", sr[0]);
Assert.AreEqual("c", sr[1]);
Assert.AreEqual("d", sr[2]);
#endregion
#region list
r.ListLeftPush("list", "1");
r.ListLeftPush("list", "2");
r.ListLeftPush("list", "3");
r.ListLeftPush("list", "4");
r.ListRightPush("list", "5");
Assert.AreEqual(5, r.ListLength("list"));
Assert.AreEqual("4", r.ListLeftPop("list"));
Assert.AreEqual("5", r.ListRightPop("list"));
Assert.AreEqual("1", r.ListRightPop("list"));
Assert.AreEqual("3", r.ListLeftPop("list"));
#endregion
#region locker
r.LockTake("locker", DateTime.Now.ToString(), new TimeSpan(0, 0, 0, 30));
#endregion
r.FlushDb();
r.Dispose();
}
public void RedisBaseTest()
{
RedisServer r = new RedisServer("localhost");
r.Db = 3;
#region set
long i;
r.SetLength("a");
r.SetAdd("foo", "bar");
r.FlushDb();
Assert.IsTrue((i = r.SetLength("foo")) == 0, "there should be no keys but there were {0}", i);
r.SetAdd("foo", "bar");
Assert.IsTrue((i = r.SetLength("foo")) == 1, "there should be one key but there were {0}", i);
r.SetAdd("foo bär", "bär foo");
r.SetAdd("foo", "bär foo");
Assert.IsTrue((i = r.SetLength("foo")) == 2, "there should be two keys but there were {0}", i);
Assert.IsTrue(r.TypeOf("foo") == KeyType.Set, "type is not string");
r.SetAdd("bar", "foo");
Assert.IsTrue(r.SetContains("bar", "foo"));
var mems = r.SetMembers("foo");
Assert.AreEqual("bar", mems[0]);
Assert.AreEqual("bär foo", mems[1]);
r.SetRemove("foo", "bar");
mems = r.SetMembers("foo");
Assert.AreEqual("bär foo", mems[0]);
Assert.AreEqual(1, mems.Count);
string item = r.SetPop("foo");
Assert.AreEqual("bär foo", item);
Assert.AreEqual(0, r.SetLength("foo"));
#endregion
#region hasset
r.HashSet("set", "a", "a");
r.HashSet("set", "b", "b");
r.HashSet("set", "c", "c");
var hash = r.HashGetAll("set");
Assert.AreEqual(hash.Length, 3);
r.HashDelete("set", "a");
hash = r.HashGetAll("set");
Assert.AreEqual(hash.Length, 2);
Assert.AreEqual(true, r.HashExists("set", "b"));
Assert.AreEqual("b", r.HashGet("set", "b"));
Assert.AreEqual(2, r.HashLength("set"));
#endregion
#region storedset
r.SortedSetAdd("sortedset", "a", 0L);
r.SortedSetAdd("sortedset", "b", 0L);
r.SortedSetAdd("sortedset", "c", 0L);
r.SortedSetAdd("sortedset", "d", 0L);
Assert.AreEqual(4, r.SortedSetLength("sortedset"));
r.SortedSetRemove("sortedset", "a");
Assert.AreEqual(3, r.SortedSetLength("sortedset"));
string[] sr = r.SortedSetRangeByRank("sortedset", 0, 2);
Assert.AreEqual("b", sr[0]);
Assert.AreEqual("c", sr[1]);
Assert.AreEqual("d", sr[2]);
#endregion
#region list
r.ListLeftPush("list", "1");
r.ListLeftPush("list", "2");
r.ListLeftPush("list", "3");
r.ListLeftPush("list", "4");
r.ListRightPush("list", "5");
Assert.AreEqual(5, r.ListLength("list"));
Assert.AreEqual("4", r.ListLeftPop("list"));
Assert.AreEqual("5", r.ListRightPop("list"));
Assert.AreEqual("1", r.ListRightPop("list"));
Assert.AreEqual("3", r.ListLeftPop("list"));
#endregion
#region locker
r.LockTake("locker", DateTime.Now.ToString(), new TimeSpan(0, 0, 0, 30));
#endregion
r.FlushDb();
r.Dispose();
}