public static void Run(object obj)
{
Tuple <int, int> tuple = (Tuple <int, int>)obj;
int partition = tuple.Item1, clientIndex = tuple.Item2;
IRedisClient redisClient = clients[clientIndex];
// PinThreadOnCores(clientIndex);
// debug
Console.WriteLine("{0}:{1}", redisClient.Host, redisClient.Port);
SLIM.Wait();
long beginTicks = DateTime.Now.Ticks;
for (int i = 0; i < BATCHES; i++)
{
using (IRedisPipeline pipe = redisClient.CreatePipeline())
{
string hashId = StaticRandom.RandIdentity().ToString();
byte[] key = BitConverter.GetBytes(StaticRandom.RandIdentity());
for (int j = 0; j < 100; j++)
{
pipe.QueueCommand(r => ((RedisNativeClient)r).HGet(hashId, key));
}
pipe.Flush();
}
}
long endTicks = DateTime.Now.Ticks;
int throughput = (int)(BATCHES * 100 * 1.0 / ((endTicks - beginTicks) * 1.0 / 10000000));
// Console.WriteLine("Single Thread Throughput: {0}", throughput);
}
/// <inheritdoc />
public IEnumerable <ITaskRuntimeInfo> GetPending(bool includePollingQueueTasks)
{
Trace.WriteLine("ENTER: Getting runtime information for pending tasks ({0} polling queue tasks).".FormatInvariant(includePollingQueueTasks ? "include" : "without"));
List <ITaskRuntimeInfo> result = this.GetAll(RedisTaskRuntimeInfoRepository.PendingTasksList);
if (includePollingQueueTasks)
{
var listKeys = this.provider.SearchKeys(RedisTaskRuntimeInfoRepository.PendingTasksList + "$*");
List <string> entityIds = new List <string>();
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
foreach (string listKey in listKeys)
{
pipeline.GetList(listKey, values => entityIds.AddRange(values));
}
pipeline.Flush();
}
result.AddRange(this.GetAll(entityIds).Where(t => t.Status == TaskStatus.Pending));
}
Trace.WriteLine("EXIT: Return runtime information for {0} pending tasks ({1} polling queue tasks).".FormatInvariant(result.Count, includePollingQueueTasks ? "include" : "without"));
return(result);
}
/// <inheritdoc />
public IEnumerable <ITaskProcessorRuntimeInfo> GetAll()
{
IEnumerable <string> entityIds = this.provider.GetSet(RedisTaskProcessorRuntimeInfoRepository.EntitySetKey);
List <ITaskProcessorRuntimeInfo> result = new List <ITaskProcessorRuntimeInfo>();
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
foreach (string entityId in entityIds)
{
string entityKey = RedisTaskProcessorRuntimeInfoRepository.GetEntityKey(entityId);
pipeline.GetHash(entityKey, values =>
{
if (values.Count > 0)
{
result.Add(RedisTaskProcessorRuntimeInfoRepository.Convert(values));
}
});
}
pipeline.Flush();
}
return(result);
}
/// <inheritdoc />
public IScheduledTask GetById(Guid scheduledTaskId)
{
Trace.WriteLine("ENTER: Getting scheduled task with ID '{0}' ...".FormatInvariant(scheduledTaskId));
string taskIdAsString = RedisConverter.ToString(scheduledTaskId);
IScheduledTask result;
byte[] scheduledTaskContent = null;
byte[] recurrenceDefinitionContent = null;
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
pipeline.GetHashBinaryValue(RedisScheduledTaskRepository.ScheduledTasksHashKey, taskIdAsString + "$Content", value => scheduledTaskContent = value);
pipeline.GetHashBinaryValue(RedisScheduledTaskRepository.ScheduledTasksHashKey, taskIdAsString + "$RecurrenceDefinition", value => recurrenceDefinitionContent = value);
if (this.serializer.CanDetermineEntityTypeFromContent)
{
pipeline.Flush();
if ((scheduledTaskContent == null) || (scheduledTaskContent.Length == 0) || (recurrenceDefinitionContent == null) || (recurrenceDefinitionContent.Length == 0))
{
return(null);
}
result = (IScheduledTask)this.serializer.Deserialize(scheduledTaskContent);
result.Schedule = (IScheduleDefinition)this.serializer.Deserialize(recurrenceDefinitionContent);
}
else
{
string scheduledTaskType = null;
string recurrenceDefinitionType = null;
pipeline.GetHashTextValue(RedisScheduledTaskRepository.ScheduledTasksHashKey, taskIdAsString + "$Type", value => scheduledTaskType = value);
pipeline.GetHashTextValue(RedisScheduledTaskRepository.ScheduledTasksHashKey, taskIdAsString + "$RecurrenceDefinition$Type", value => recurrenceDefinitionType = value);
pipeline.Flush();
result = this.Deserialize(scheduledTaskId.ToString(), scheduledTaskContent, scheduledTaskType, recurrenceDefinitionContent, recurrenceDefinitionType);
}
}
Trace.WriteLine("EXIT: Scheduled task '{0}' with ID '{1}' returned.".FormatInvariant(result, scheduledTaskId));
return(result);
}
private void LeavePipelining()
{
if (Interlocked.CompareExchange(ref _pipelineMode, 0, 1) == 1)
{
_pipeline.Flush();
_pipeline.Dispose();
}
}
internal override void Clear()
{
if (this.redisVersionDbMode == RedisVersionDbMode.Cluster)
{
// IMPORTMENT: Since the Redis Cluster doesn't allow multi-key commands across multiple hash slots
// So we couldn't clear keys in batch
//using (RedisClient redisClient = this.singletonConnPool.GetRedisClient())
//{
// byte[][] keysAndArgs =
// {
// Encoding.ASCII.GetBytes(RedisVersionDb.VER_KEY_PREFIX),
// };
// string sha1 = this.LuaManager.GetLuaScriptSha1(LuaScriptName.REMOVE_KEYS_WITH_PREFIX);
// redisClient.EvalSha(sha1, 0, keysAndArgs);
//}
int batchSize = 100;
using (RedisClient redisClient = this.singletonConnPool.GetRedisClient())
{
byte[][] keys = redisClient.Keys(RedisVersionDb.VER_KEY_PREFIX + "*");
if (keys != null)
{
for (int i = 0; i < keys.Length; i += batchSize)
{
int upperBound = Math.Min(keys.Length, i + batchSize);
using (IRedisPipeline pipe = redisClient.CreatePipeline())
{
for (int j = i; j < upperBound; j++)
{
string keyStr = Encoding.ASCII.GetString(keys[j]);
pipe.QueueCommand(r => ((RedisNativeClient)r).Del(keyStr));
}
pipe.Flush();
}
}
}
}
}
else
{
for (int pid = 0; pid < this.PartitionCount; pid++)
{
using (RedisClient redisClient = this.RedisManager.GetClient(
this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pid)))
{
redisClient.FlushDb();
}
}
}
}
private Dictionary <TaskStatus, IEnumerable <ITaskRuntimeInfo> > GetAllByType(params string[] listKeys)
{
List <string> entityIds = new List <string>();
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
foreach (string listKey in listKeys)
{
pipeline.GetList(listKey, values => entityIds.AddRange(values));
}
pipeline.Flush();
}
Dictionary <TaskStatus, IEnumerable <ITaskRuntimeInfo> > result = new Dictionary <TaskStatus, IEnumerable <ITaskRuntimeInfo> >();
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
foreach (string entityId in entityIds)
{
string entityKey = RedisTaskRuntimeInfoRepository.GetEntityKey(entityId);
pipeline.GetHash(entityKey, values =>
{
if (values.Count > 0)
{
ITaskRuntimeInfo taskInfo = RedisTaskRuntimeInfoRepository.Convert(values);
IEnumerable <ITaskRuntimeInfo> collection;
if (!result.TryGetValue(taskInfo.Status, out collection))
{
collection = new List <ITaskRuntimeInfo>();
result.Add(taskInfo.Status, collection);
}
((ICollection <ITaskRuntimeInfo>)collection).Add(taskInfo);
}
});
}
pipeline.Flush();
}
return(result);
}
/// <inheritdoc />
public IEnumerable <ITaskRuntimeInfo> ReservePollingQueueTasks(string pollingQueueKey, int maxResults)
{
Trace.WriteLine("ENTER: Reserving {0} polling queue '{1}' tasks for execution ...".FormatInvariant(maxResults, pollingQueueKey));
if (string.IsNullOrEmpty(pollingQueueKey))
{
throw new ArgumentNullException(nameof(pollingQueueKey));
}
if (maxResults < 0)
{
throw new ArgumentOutOfRangeException(nameof(maxResults), maxResults, "Value must not be negative.");
}
string pollingQueueListKey = RedisTaskRuntimeInfoRepository.GetPollingQueueRedisKey(pollingQueueKey, TaskStatus.Pending);
List <string> entityIds = new List <string>();
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
for (int i = 0; i < maxResults; i++)
{
pipeline.PopFirstListElementAsText(pollingQueueListKey, value =>
{
if (!string.IsNullOrEmpty(value))
{
entityIds.Add(value);
}
});
}
pipeline.Flush();
}
ICollection <ITaskRuntimeInfo> result = this.GetAll(entityIds);
Trace.WriteLine("EXIT: {0} polling queue '{1}' tasks reserved for execution.".FormatInvariant(result.Count, pollingQueueKey));
return(result);
}
/// <inheritdoc />
public void Set(Type taskType, ITaskJobSettings settings)
{
Trace.WriteLine("ENTER: Setting task job settings '{0}' for task '{1}' ...".FormatInvariant(settings, taskType));
if (taskType == null)
{
throw new ArgumentNullException(nameof(taskType));
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
if (!typeof(ITask).IsAssignableFrom(taskType))
{
throw new ArgumentException("Type '{0}' does not implement '{1}'.".FormatInvariant(taskType, typeof(ITask), nameof(taskType)));
}
byte[] content = this.serializer.Serialize(settings);
if (this.serializer.CanDetermineEntityTypeFromContent)
{
this.provider.SetHashValue(RedisTaskJobSettingsRepository.RedisTaskJobSettingsHashKey, taskType.Name, content);
}
else
{
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
pipeline.SetHashValue(RedisTaskJobSettingsRepository.RedisTaskJobSettingsHashKey, taskType.Name, content);
pipeline.SetHashValue(RedisTaskJobSettingsRepository.RedisTaskJobSettingsHashKey, taskType.Name + "$Type", RedisConverter.ToString(settings.GetType(), false));
pipeline.Flush();
}
}
Trace.WriteLine("EXIT: Task job settings '{0}' for task '{1}' set.".FormatInvariant(settings, taskType));
}
/// <inheritdoc />
public IEnumerable <ITaskRuntimeInfo> GetFailed()
{
Trace.WriteLine("ENTER: Getting runtime information for failed tasks ...");
IEnumerable <string> entityIds = this.provider.GetListAsText(RedisTaskRuntimeInfoRepository.FailedTasksList);
List <ITaskRuntimeInfo> result = new List <ITaskRuntimeInfo>();
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
foreach (string taskId in entityIds)
{
pipeline.GetHashBinaryValue(RedisTaskRuntimeInfoRepository.ArchiveTasksHash, taskId, content =>
result.Add((ITaskRuntimeInfo)this.serializer.Deserialize(content, typeof(RedisTaskRuntimeInfo))));
}
pipeline.Flush();
}
Trace.WriteLine("EXIT: Return runtime information for {0} failed tasks.".FormatInvariant(result.Count));
return(result);
}
static void LocalRedisBenchmarkTest()
{
int count = 100000, pipelineSize = 100;
Random rand = new Random();
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
Func <int, string> RandomString = (int length) =>
{
return(new string(Enumerable.Repeat(chars, length)
.Select(s => s[rand.Next(s.Length)]).ToArray()));
};
Func <int, byte[]> RandomBytes = (int length) =>
{
byte[] value = new byte[length];
rand.NextBytes(value);
return(value);
};
RedisVersionDb redisVersionDb = RedisVersionDb.Instance();
// Non-Pipeline Mode
using (RedisClient client = redisVersionDb.RedisManager.GetClient(3, 0))
{
long now = DateTime.Now.Ticks;
for (int i = 0; i < count; i++)
{
string hashId = RandomString(3);
byte[] key = BitConverter.GetBytes(3);
byte[] value = RandomBytes(50);
int type = rand.Next(0, 1);
if (type == 0)
{
client.HSet(hashId, key, value);
}
else
{
client.HGet(hashId, key);
}
}
long time = DateTime.Now.Ticks - now;
int throughput = (int)((count * 1.0) / (time * 1.0 / 10000000));
Console.WriteLine("Redis Local Non Pipeline Throughput: {0} ops/s", throughput);
}
// Pipeline Mode
using (RedisClient client = redisVersionDb.RedisManager.GetClient(3, 0))
{
long now = DateTime.Now.Ticks;
int i = 0;
while (i < count)
{
using (IRedisPipeline pipeline = client.CreatePipeline())
{
for (int j = 0; j < pipelineSize; j++)
{
string hashId = RandomString(3);
byte[] key = BitConverter.GetBytes(3);
byte[] value = RandomBytes(50);
int type = rand.Next(0, 1);
if (type == 0)
{
pipeline.QueueCommand(
r => ((RedisNativeClient)r).HSet(hashId, key, value));
}
else
{
pipeline.QueueCommand(
r => ((RedisNativeClient)r).HGet(hashId, key));
}
}
pipeline.Flush();
}
i += pipelineSize;
}
long time = DateTime.Now.Ticks - now;
int throughput = (int)((count * 1.0) / (time * 1.0 / 10000000));
Console.WriteLine("Redis Local Pipeline({0}) Throughput: {1} ops/s", pipelineSize, throughput);
}
}
/// <inheritdoc />
public ITaskJobSettings Get(Type taskType)
{
Trace.WriteLine("ENTER: Getting task job settings for task '{0}' ...".FormatInvariant(taskType));
if (taskType == null)
{
throw new ArgumentNullException(nameof(taskType));
}
if (!typeof(ITask).IsAssignableFrom(taskType))
{
throw new ArgumentException("Type '{0}' does not implement '{1}'.".FormatInvariant(taskType, typeof(ITask), nameof(taskType)));
}
ITaskJobSettings result;
if (this.serializer.CanDetermineEntityTypeFromContent)
{
byte[] content = this.provider.GetHashBinaryValue(RedisTaskJobSettingsRepository.RedisTaskJobSettingsHashKey, taskType.Name);
result = (ITaskJobSettings)this.serializer.Deserialize(content);
}
else
{
byte[] content = null;
string settingsTypeAsString = null;
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
pipeline.GetHashBinaryValue(RedisTaskJobSettingsRepository.RedisTaskJobSettingsHashKey, taskType.Name, value => content = value);
pipeline.GetHashTextValue(RedisTaskJobSettingsRepository.RedisTaskJobSettingsHashKey, taskType.Name + "$Type", value => settingsTypeAsString = value);
pipeline.Flush();
}
if ((content == null) || (content.Length == 0))
{
return(null);
}
if (string.IsNullOrEmpty(settingsTypeAsString))
{
#if DEBUG
throw new TypeNotFoundInRedisException("Task job settings type for task '{0}' was not found in Redis.".FormatInvariant(taskType));
#else
Trace.TraceWarning("EXIT: Task job settings type for task '{0}' was not found in Redis.".FormatInvariant(taskType));
return(null);
#endif
}
Type settingsType;
#if DEBUG
settingsType = Type.GetType(settingsTypeAsString, true);
#else
settingsType = Type.GetType(settingsTypeAsString, false);
if (settingsType == null)
{
Trace.TraceWarning("EXIT: Task job settings type '{0}' cannot be resolved.", settingsTypeAsString);
return(null);
}
#endif
result = (ITaskJobSettings)this.serializer.Deserialize(content, settingsType);
}
Trace.WriteLine("EXIT: Return task job settings '{0}' for task '{1}'.".FormatInvariant(result, taskType));
return(result);
}
internal void Flush(IEnumerable <RedisRequest> requests, RedisClient redisClient = null, int maxRequests = -1)
{
int reqCount = 0;
using (IRedisPipeline pipe = redisClient.CreatePipeline())
{
foreach (RedisRequest req in requests)
{
reqCount++;
if (req == null)
{
continue;
}
switch (req.Type)
{
case RedisRequestType.HGet:
pipe.QueueCommand(
r => ((RedisNativeClient)r).HGet(req.HashId, req.Key),
req.SetValue, req.SetError);
break;
case RedisRequestType.HMGet:
pipe.QueueCommand(
r => ((RedisNativeClient)r).HMGet(req.HashId, req.Keys),
req.SetValues, req.SetError);
break;
case RedisRequestType.HGetAll:
pipe.QueueCommand(
r => ((RedisNativeClient)r).HGetAll(req.HashId),
req.SetValues, req.SetError);
break;
case RedisRequestType.HSetNX:
pipe.QueueCommand(
r => ((RedisNativeClient)r).HSetNX(req.HashId, req.Key, req.Value),
req.SetLong, req.SetError);
break;
case RedisRequestType.HSet:
pipe.QueueCommand(
r => ((RedisNativeClient)r).HSet(req.HashId, req.Key, req.Value),
req.SetLong, req.SetError);
break;
case RedisRequestType.HMSet:
pipe.QueueCommand(
r => ((RedisNativeClient)r).HMSet(req.HashId, req.Keys, req.Values),
req.SetVoid, req.SetError);
break;
case RedisRequestType.HDel:
// delete a single field
if (req.Key != null)
{
pipe.QueueCommand(
r => ((RedisNativeClient)r).HDel(req.HashId, req.Key),
req.SetLong, req.SetError);
}
// delete multiple fields
else
{
pipe.QueueCommand(
r => ((RedisNativeClient)r).HDel(req.HashId, req.Keys),
req.SetLong, req.SetError);
}
break;
case RedisRequestType.EvalSha:
pipe.QueueCommand(
r => ((RedisNativeClient)r).EvalSha(req.Sha1, req.NumberKeysInArgs, req.Keys),
req.SetValues, req.SetError);
break;
default:
break;
}
if (maxRequests != -1 && reqCount >= maxRequests)
{
break;
}
}
pipe.Flush();
}
}
/// <inheritdoc />
public ITaskSummary GetById(Guid taskId)
{
Trace.WriteLine("ENTER: Getting summary for task '{0}' ...".FormatInvariant(taskId));
string taskIdAsString = RedisConverter.ToString(taskId);
byte[] content = null;
ITaskSummary result;
if (this.serializer.CanDetermineEntityTypeFromContent)
{
content = this.provider.GetHashBinaryValue(RedisTaskSummaryRepository.RedisTaskSummaryHashKey, taskIdAsString);
result = (ITaskSummary)this.serializer.Deserialize(content);
}
else
{
string summaryTypeAsString = null;
using (IRedisPipeline pipeline = this.provider.CreatePipeline())
{
pipeline.GetHashBinaryValue(RedisTaskSummaryRepository.RedisTaskSummaryHashKey, taskIdAsString, value => content = value);
pipeline.GetHashTextValue(RedisTaskSummaryRepository.RedisTaskSummaryHashKey, taskIdAsString + "$Type", value => summaryTypeAsString = value);
pipeline.Flush();
}
if ((content == null) || (content.Length == 0))
{
return(null);
}
if (string.IsNullOrEmpty(summaryTypeAsString))
{
#if DEBUG
throw new TypeNotFoundInRedisException("Task summary type for task '{0}' was not found in Redis.".FormatInvariant(taskId));
#else
Trace.TraceWarning("EXIT: Task summary type for task '{0}' was not found in Redis.".FormatInvariant(taskId));
return(null);
#endif
}
Type summaryType;
#if DEBUG
summaryType = Type.GetType(summaryTypeAsString, true);
#else
summaryType = Type.GetType(summaryTypeAsString, false);
if (summaryType == null)
{
Trace.TraceWarning("EXIT: Task summary type '{0}' cannot be resolved.", summaryTypeAsString);
return(null);
}
#endif
result = (ITaskSummary)this.serializer.Deserialize(content, summaryType);
}
Trace.WriteLine("EXIT: Return task summary '{0}' for task '{1}'.".FormatInvariant(result, taskId));
return(result);
}
internal override void MockLoadData(int recordCount)
{
int pk = 0;
RedisConnectionPool connPool = null;
if (this.redisVersionDbMode == RedisVersionDbMode.Cluster)
{
connPool = this.singletonConnPool;
}
int loaded = 0;
while (pk < this.PartitionCount)
{
Console.WriteLine("Loading Partition {0}", pk);
if (connPool == null)
{
connPool = this.RedisManager.GetClientPool(this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk));
}
using (RedisClient redisClient = connPool.GetRedisClient())
{
int batchSize = 100;
int partitions = this.PartitionCount;
for (int i = pk; i < recordCount; i += partitions * batchSize)
{
int upperBound = Math.Min(recordCount, i + partitions * batchSize);
using (IRedisPipeline pipe = redisClient.CreatePipeline())
{
for (int j = i; j < upperBound; j += partitions)
{
object recordKey = j;
string hashId = recordKey.ToString();
if (this.redisVersionDbMode == RedisVersionDbMode.Cluster)
{
hashId = RedisVersionDb.PACK_KEY(RedisVersionDb.VER_KEY_PREFIX, hashId);
}
VersionEntry versionEntry = new VersionEntry();
VersionEntry.InitFirstVersionEntry(new String('a', 100), versionEntry);
byte[] key = BitConverter.GetBytes(VersionEntry.VERSION_KEY_START_INDEX + 1);
byte[] value = VersionEntry.Serialize(versionEntry);
pipe.QueueCommand(r => ((RedisNativeClient)r).HSet(hashId, key, value));
pipe.QueueCommand(r => ((RedisNativeClient)r).HSet(hashId, RedisVersionDb.LATEST_VERSION_PTR_FIELD, key));
loaded++;
}
pipe.Flush();
}
}
}
pk++;
if (this.redisVersionDbMode != RedisVersionDbMode.Cluster)
{
connPool = null;
}
}
Console.WriteLine("Loaded {0} records Successfully", loaded);
}