public static Position RedisTryGetPosition(RedisDataAccessProvider redis, string key)
{
int commandId = redis.SendCommand(RedisCommand.GET, key);
string positionData = Utils.RedisTryReadString(redis, commandId);
return(JSONTo <Position>(positionData));
}
public static Texture2D GetImageAsTexture(RedisDataAccessProvider redis, string key)
{
int commandId = redis.SendCommand(RedisCommand.GET, key + ":width");
int?width = Utils.RedisTryReadInt(redis, commandId);
commandId = redis.SendCommand(RedisCommand.GET, key + ":height");
int?height = Utils.RedisTryReadInt(redis, commandId);
commandId = redis.SendCommand(RedisCommand.GET, key + ":channels");
int?channels = Utils.RedisTryReadInt(redis, commandId);
if (width == null || height == null || channels == null)
{
return(null);
}
Texture2D texture = new Texture2D((int)width, (int)height, TextureFormat.RGB24, false, false);
byte[] imageData = new byte[(int)width * (int)height * (int)channels];
bool succeed = GetImageIntoPreallocatedTexture(redis, key, ref texture, imageData, (int)width, (int)height, (int)channels);
if (succeed)
{
return(texture);
}
else
{
return(null);
}
}
public Subscriber(RedisDataAccessProvider redis)
{
RedisConnection connection = new RedisConnection(redis.Configuration.Host, redis.Configuration.Port);
this.redis = connection.GetDataAccessProvider();
this.redis.ChannelSubscribed += new ChannelSubscribedHandler(OnChannelSubscribed);
}
public oKey(RedisDataAccessProvider redis, string s)
{
string[] a = s.Split(new char[] { '#', '.' });
this.key_full = s;
this.key0 = a[0];
this.level = 0;
if (a.Length > 1)
{
this.key1 = a[1];
this.level = 1;
}
if (a.Length > 2)
{
this.key2 = a[2];
this.level = 2;
}
if (a.Length > 3)
{
this.key3 = a[3];
this.level = 3;
}
try
{
string[] ks = redis.ReadMultiString(redis.SendCommand(RedisCommand.HKEYS, s));
this.keys_last = ks.OrderBy(x => x).ToArray();
}
catch (Exception e)
{
}
}
public void MultiThead()
{
var threads = new List <Thread>();
redis.Configuration.LogUnbalancedCommands = false;
redis.WaitComplete(redis.SendCommand(RedisCommand.FLUSHALL));
for (int x = 0; x < 3; x++)
{
threads.Add(new Thread(() =>
{
Random rnd = new Random();
try
{
for (int i = 0; i < 1000; i++)
{
Guid value = Guid.NewGuid();
string key = string.Format("test_{0}", value.ToString());
var r = new RedisDataAccessProvider();
r.Key.Exists(key);
r.Strings[key].Set(value.ToString());
r.Key.Exists(key);
Assert.AreEqual(value.ToString(), r.Strings[key].Get());
r.Key.Exists(string.Format("test_{0}", rnd.Next(10000)));
Debug.WriteLine(string.Format("Thread {0} -> {1} ", Thread.CurrentThread.ManagedThreadId, i));
}
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}));
}
foreach (var t in threads)
{
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(CurrentDomain_UnhandledException);
}
foreach (var t in threads)
{
t.Start();
}
foreach (var t in threads)
{
t.Join();
}
redis.WaitComplete(redis.SendCommand(RedisCommand.FLUSHALL));
}
public static ExtrinsicsParameters RedisTryGetExtrinsics(RedisDataAccessProvider redis, string key)
{
int commandId = redis.SendCommand(RedisCommand.GET, key);
string extrinsicsData = null;
extrinsicsData = Utils.RedisTryReadString(redis, commandId);
return(JSONTo <ExtrinsicsParameters>(extrinsicsData));
}
public async Task Connect(string ipAddress, int port = 6379)
{
await Task.Run(() => {
client = new RedisDataAccessProvider();
client.Configuration.Host = ipAddress;
client.Configuration.Port = port;
client.Connect();
});
}
private void updateParentConnection()
{
parent = new RedisDataAccessProvider();
parent.Configuration.Host = RedisServerHost;
parent.Configuration.Port = RedisServerPort;
parent.Configuration.NoDelaySocket = NoDelaySocket;
ConnectionStatusChanged += OnConnectionStatusChanged;
NewService += OnNewServiceArrived;
}
/// <summary>
/// Tries to read redis command output as raw data (byte)
/// </summary>
/// <param name="redis"></param>
/// <param name="commandId">the command to read the output from</param>
/// <returns></returns>
public static byte[] RedisTryReadData(RedisDataAccessProvider redis, int commandId)
{
byte[] data = null;
try {
data = redis.ReadData(commandId);
} catch (Exception) {
Debug.Log("Failed to read data.");
}
return(data);
}
private void connect()
{
redis = rHandler.CreateConnection();
try {
redis.Connect();
} catch (Exception) {
OnServiceConnectionStateChanged(false);
return;
}
OnServiceConnectionStateChanged(true);
}
/// <summary>
/// Tries to read redis command output as a string.
/// </summary>
/// <param name="redis"></param>
/// <param name="commandId">the command to read the output</param>
/// <returns>the string if success, else null.</returns>
public static string RedisTryReadString(RedisDataAccessProvider redis, int commandId)
{
string value = null;
try {
value = redis.ReadString(commandId);
} catch (Exception) {
Debug.LogError("Failed to read string.");
}
return(value);
}
/// <summary>
/// Tries to get 3D pose information saved in redis
/// </summary>
/// <param name="key">the key whereto look for 3d pose"</param>
/// <returns></returns>
public static Matrix4x4?RedisTryGetPose3D(RedisDataAccessProvider redis, string key)
{
int commandId = redis.SendCommand(RedisCommand.GET, key);
string jsonPose = Utils.RedisTryReadString(redis, commandId);
if (jsonPose != null)
{
// Hand made parsing
return(JSONToPose3D(jsonPose));
}
return(null);
}
/// <summary>
/// Tries to read redis command output as an int.
/// </summary>
/// <param name="redis"></param>
/// <param name="commandId">the command to read the output</param>
/// <returns>the int if success, else null</returns>
public static int?RedisTryReadInt(RedisDataAccessProvider redis, int commandId)
{
string value = null;
value = RedisTryReadString(redis, commandId);
int val;
if (Int32.TryParse(value, out val))
{
return(val);
}
return(null);
}
/// <summary>
/// Disconnects from the current Redis instance.
/// </summary>
public void Disconnect()
{
if (redisDap == null)
{
return;
}
// Stop, dispose and delete the current redis interaction.
redisDap.Close();
redisDap.Dispose();
redisDap = null;
}
public void Start()
{
parent = new RedisDataAccessProvider();
parent.Configuration.Host = RedisServerHost;
parent.Configuration.Port = RedisServerPort;
parent.Configuration.NoDelaySocket = NoDelaySocket;
ConnectionStatusChanged += OnConnectionStatusChanged;
NewService += OnNewServiceArrived;
Connect();
pingRoutine = PingRedis();
StartCoroutine(pingRoutine);
}
public void TestPUBSUB_Bug1()
{
RedisDataAccessProvider pp = new RedisDataAccessProvider();
RedisDataAccessProvider.ActivateDebugLog = true;
pp.ChannelSubscribed += new ChannelSubscribedHandler(pp_ChannelSubscribed);
pp.MessageReceived += new MessageReceivedHandler(pp_MessageReceived);
pp.Messaging.Subscribe("message", "message2");
pp.Messaging.Publish("message", "test1");
pp.Messaging.Publish("message", "test2");
Thread.Sleep(1000000);
}
public RedisDataAccessProvider CreateConnection()
{
if (parent == null || !redisConnected)
{
Debug.LogError("No reference connection found or Redis server is unreachable: " + redisConnected);
return(null);
}
RedisDataAccessProvider redis = new RedisDataAccessProvider();
redis.Configuration.Host = parent.Configuration.Host;
redis.Configuration.Port = parent.Configuration.Port;
redis.Configuration.NoDelaySocket = parent.Configuration.NoDelaySocket;
return(redis);
}
public void TestPUBSUB()
{
RedisDataAccessProvider pp = new RedisDataAccessProvider();
pp.ChannelSubscribed += new ChannelSubscribedHandler(pp_ChannelSubscribed);
pp.MessageReceived += new MessageReceivedHandler(pp_MessageReceived);
pp.Messaging.Subscribe("message");
pp.Messaging.Publish("message", "prova1");
pp.Messaging.Publish("message", "prova2");
pp.Messaging.Publish("message", "prova3");
pp.Messaging.Publish("message", "prova4");
pp.Messaging.Publish("message", "prova5");
pp.Messaging.Publish("message", "prova6");
Thread.Sleep(1000000);
}
public static bool GetImageIntoPreallocatedTexture(RedisDataAccessProvider redis, string key, ref Texture2D texture, byte[] textureRaw, int width, int height, int channels)
{
int commandId = redis.SendCommand(RedisCommand.GET, key);
// 70-90% of the time is spent here
textureRaw = Utils.RedisTryReadData(redis, commandId);
if (textureRaw == null)
{
return(false);
}
if (channels == 2)
{
textureRaw = Utils.GRAY16ToRGB24((int)width, (int)height, textureRaw);
}
texture.LoadRawTextureData(textureRaw);
texture.Apply();
return(true);
}
Collector(
CollectorManager cm,
string link, string backlink, string text, string rel, string kind,
bool crawlChildLinks, bool poolChildLinksFound)
{
this.CM = cm;
this.Redis = cm.Redis;
this.Link = link;
this.CurrentBacklink = LinkParser.Validate(backlink, string.Empty);
this.LinkInfo = new LinkInfo(cm, this.Link, this.CurrentBacklink, text, rel, kind);
if (!this.LinkInfo.LinkExcludedInRobots)
{
lock (o)
{
SaveLink(this);
}
if (crawlChildLinks)
CrawlChildLinks();
}
}
public RobotsCache(CollectorManager cm, string domainSchemeAndServer)
{
this.Redis = cm.Redis;
this.DomainSchemeAndServer = new Uri(domainSchemeAndServer).AbsoluteUri;
this.RobotID = "urn:domain:robots:data";
this.RobotLastDateCrawlID = "urn:domain:robots:last-date-crawl";
if (!TryRetrieveFromCache())
{
RetrieveRobots();
//this.Redis.SetEntryInHash(this.RobotID, this.DomainSchemeAndServer,
// this.RobotsExclusion.JsonSerialize());
this.Redis.Hash[this.RobotID][this.DomainSchemeAndServer] =
this.RobotsExclusion.JsonSerialize();
//this.Redis.SetEntryInHash(this.RobotLastDateCrawlID,this.DomainSchemeAndServer,
// DateTime.Now.ToString());
this.Redis.Hash[this.RobotLastDateCrawlID][this.DomainSchemeAndServer] =
DateTime.Now.ToString();
}
}
/// <summary>
/// Initializes the accessor to communicate with a Redis instance.
/// </summary>
/// <param name="hostName">Hostname of the Redis service we're hitting.</param>
/// <param name="portNum">Port number we're talking to for the Redis instance.</param>
/// <returns>If the DAO was successfully initialized.</returns>
public bool Initialize(string hostName, int portNum)
{
try
{
// Set up the accessor, and connect to the Redis instance passed.
this.redisDap = new RedisDataAccessProvider();
this.redisDap.Configuration.Host = hostName;
this.redisDap.Configuration.Port = portNum;
this.redisDap.Connect();
// Assign all extender classes.
this.Hashing = new RedisHashingAccessor(this.redisDap, this.log);
this.Sets = new RedisSetAccessor(this.redisDap, this.log);
}
catch (SocketException ex)
{
log.LogMessage($"Error talking to the Redis instance at { hostName }:{ portNum }. Can't access the Redis service. Ex: { ex.ToString() }");
this.redisDap = null;
}
return(this.redisDap != null);
}
public RedisConnection(string ip, int port)
{
redis = new RedisDataAccessProvider();
redis.Configuration.Host = IpAdress;
redis.Configuration.Port = port;
}
// ~Collector() { if (this.Redis != null) this.Redis.Dispose(); }
static void SaveLink(Collector c)
{
//IRedisHash urnLinkData = this.Redis.Hashes["urn:link:data"];
var urnLinkData = c.Redis.Hash["urn:link:data"];
if (urnLinkData.ContainsKey(c.Link))
{
string serializedLinkData = urnLinkData[c.Link];
c.LinkInfo.Merge(serializedLinkData.JsonDeserialize<LinkInfo>());
}
//me.Redis.SetEntryInHash("urn:link:data",
// me.Link, me.LinkInfo.JsonSerialize());
c.Redis.Hash["urn:link:data"][c.Link] = c.LinkInfo.JsonSerialize();
//me.Redis.SetEntryInHash("urn:link:data-last-date-crawl",
// me.Link, DateTime.Now.ToString());
c.Redis.Hash["urn:link:data-last-date-crawl"][c.Link] = DateTime.Now.ToString();
// Index date last crawl
//IRedisHash urnLinkDateLastCrawl = me.Redis.Hashes["urn:link:date-last-crawl"];
// Seperate redis connection seems to be fixing race conditions
using (RedisDataAccessProvider myRedis = new RedisDataAccessProvider())
{
myRedis.Configuration = c.Redis.Configuration;
var urnLinkDateLastCrawl = myRedis.Hash["urn:link:date-last-crawl"];
List<string> dateLastCrawlLinks = new List<string>();
if (urnLinkDateLastCrawl.ContainsKey(DateTime.Today.ToString()))
dateLastCrawlLinks = urnLinkDateLastCrawl[DateTime.Today.ToString()]
.JsonDeserialize<List<string>>();
else
dateLastCrawlLinks = new List<string>();
if (!dateLastCrawlLinks.Contains(c.Link))
dateLastCrawlLinks.Add(c.Link);
//me.Redis.SetEntryInHash("urn:link:date-last-crawl",
// DateTime.Today.ToString(), dateLastCrawlLinks.JsonSerialize());
urnLinkDateLastCrawl.Set(DateTime.Today.ToString(), dateLastCrawlLinks.JsonSerialize());
myRedis.Close();
}
if (!string.IsNullOrEmpty(c.CurrentBacklink))
{
// Index anchor
//IRedisHash urnLinkAnchor = me.Redis.Hashes["urn:link:anchor"];
var urnLinkAnchor = c.Redis.Hash["urn:link:anchor"];
List<string> anchorData;
if (urnLinkAnchor.ContainsKey(c.LinkInfo.LinkPairID))
anchorData = urnLinkData[c.LinkInfo.LinkPairID]
.JsonDeserialize<List<string>>();
else
anchorData = new List<string>();
if (!anchorData.Contains(c.LinkInfo.AnchorInfo.JsonSerialize()))
{
anchorData.Add(c.LinkInfo.AnchorInfo.JsonSerialize());
//me.Redis.SetEntryInHash("urn:link:anchor",
// me.LinkInfo.LinkPairID, anchorData.JsonSerialize());
urnLinkAnchor[c.LinkInfo.LinkPairID] = anchorData.JsonSerialize();
}
//Redis.Lists["urn:link:anchor:" + me.LinkInfo.LinkPairID].RemoveValue(me.LinkInfo.AnchorInfo.JsonSerialize());
//Redis.Lists["urn:link:anchor:" + me.LinkInfo.LinkPairID].Append(me.LinkInfo.AnchorInfo.JsonSerialize());
// TODO: Index external backlinks
//foreach (string backlink in me.LinkInfo.Backlinks)
//{
//Redis.Lists["urn:backlink-external-link:" + backlink.Replace(':','_')].RemoveValue(me.Link);
//Redis.Lists["urn:backlink-external-link:" + backlink.Replace(':', '_')].Append(me.Link);
//}
// Index backlink count by link
//me.Redis.SetEntryInHash("urn:link:backlink-count",
// me.Link, me.LinkInfo.Backlinks.Count.ToString());
c.Redis.Hash["urn:link:backlink-count"][c.Link] =
c.LinkInfo.Backlinks.Count.ToString();
}
// Index domain or subdomain
//IRedisHash urnLinkDomainOrSubdomain = me.Redis.Hashes["urn:link:domain-or-subdomain"];
var urnLinkDomainOrSubdomain = c.Redis.Hash["urn:link:domain-or-subdomain"];
List<string> domainOrSubdomainLinks;
if (urnLinkDomainOrSubdomain.ContainsKey(c.LinkInfo.DomainOrSubdomain))
domainOrSubdomainLinks = urnLinkDomainOrSubdomain[c.LinkInfo.DomainOrSubdomain]
.JsonDeserialize<List<string>>();
else
domainOrSubdomainLinks = new List<string>();
if (!domainOrSubdomainLinks.Contains(c.Link))
domainOrSubdomainLinks.Add(c.Link);
//me.Redis.SetEntryInHash("urn:link:domain-or-subdomain",
// me.LinkInfo.DomainOrSubdomain, domainOrSubdomainLinks.JsonSerialize());
urnLinkDomainOrSubdomain[c.LinkInfo.DomainOrSubdomain] =
domainOrSubdomainLinks.JsonSerialize();
// Index domain
//IRedisHash urnLinkDomain = me.Redis.Hashes["urn:link:domain"];
var urnLinkDomain = c.Redis.Hash["urn:link:domain"];
List<string> domainLinks;
if (urnLinkDomain.ContainsKey(c.LinkInfo.Domain))
domainLinks = urnLinkDomain[c.LinkInfo.Domain]
.JsonDeserialize<List<string>>();
else
domainLinks = new List<string>();
if (!domainLinks.Contains(c.Link))
domainLinks.Add(c.Link);
//me.Redis.SetEntryInHash("urn:link:domain",
// me.LinkInfo.Domain, domainLinks.JsonSerialize());
urnLinkDomain[c.LinkInfo.Domain] = domainLinks.JsonSerialize();
//Redis.Lists["urn:link:domain-or-subdomain:" + me.LinkInfo.DomainOrSubdomain].RemoveValue(me.Link);
//Redis.Lists["urn:link:domain-or-subdomain:" + me.LinkInfo.DomainOrSubdomain].Append(me.Link);
//Redis.Lists["urn:link:domain:" + me.LinkInfo.Domain].RemoveValue(me.Link);
//Redis.Lists["urn:link:domain:" + me.LinkInfo.Domain].Append(me.Link);
}
public void Start(string redisServer)
{
// Initialize and start collectors only once
if (this.Redis == null)
{
//this.PRCM = new BasicRedisClientManager(redisServer);
//this.Redis = new RedisClient(redisServer);
this.Redis = new RedisDataAccessProvider();
this.Redis.Configuration = new TeamDev.Redis.LanguageItems.Configuration();
this.Redis.Configuration.Host = redisServer;
if (CrawlNextLinkQueueManager == null)
{
CrawlNextLinkQueueManager = new System.Timers.Timer(2000);
CrawlNextLinkQueueManager.Elapsed += (o, ea) =>
{
CrawlNextLinkFromPool(this);
};
}
CrawlNextLinkQueueManager.Start();
//CrawlNextLinkFromPool(this);
if (CollectorsManager == null)
{
CollectorsManager = new System.Timers.Timer(500);
CollectorsManager.Elapsed += (o, ea) =>
{
LinkCollectorTaskPair workerToTerminate = null;
foreach (var worker in Collectors.Values)
{
if (worker.Task.IsCompleted ||
worker.Task.IsFaulted ||
worker.Task.IsCanceled)
{
workerToTerminate = worker;
break;
}
}
if (workerToTerminate != null)
{
workerToTerminate.Task.Dispose();
Collectors.Remove(workerToTerminate.LinkInfo.Link);
LinksCurrentlyProcessing.Remove(workerToTerminate.LinkInfo.Link);
CollectorCount[0] = Collectors.Count;
}
};
}
CollectorsManager.Start();
//if (PoolManager == null)
//{
// PoolManager = new System.Timers.Timer(10000);
// PoolManager.Elapsed += (o, ea) =>
// {
// PoolTop100SitesThatHaveBacklinksGreaterThan10(this);
// };
//}
//PoolManager.Stop();
//PoolManager.Start();
}
}
void ILanguageItem.Configure(string name, RedisDataAccessProvider provider)
{
_name = name;
_provider = provider;
}
void f_load(bool cache_selected = false)
{
f_message();
treeKeys.Nodes.Clear();
listKeys.Items.Clear();
lblKeyID_Selected.Text = "";
lblKeyID_Counter.Text = "";
string key_full_selected = "";
if (cache_selected && m_key_selected != null)
{
key_full_selected = m_key_selected.key_full;
}
try
{
m_redis = new RedisDataAccessProvider();
m_redis.Configuration = new Configuration()
{
Host = txtIP.Text.Trim(), Port = Convert.ToInt16(txtPort.Text.Trim())
};
m_redis.Connect();
string[] keys = m_redis.ReadMultiString(m_redis.SendCommand(RedisCommand.KEYS, "*"));
m_keys = keys.Select(x => new oKey(m_redis, x)).ToArray();
var keys_0 = keys.Select(x => x.Split(new char[] { '#', '.' }))
.Select(x => x[0]).Distinct().OrderBy(x => x).ToArray();
TreeNode[] nodes_0 = new TreeNode[keys_0.Length];
for (var i = 0; i < keys_0.Length; i++)
{
string key_0 = keys_0[i];
nodes_0[i] = new TreeNode(key_0);
var okeys_1 = m_keys.Where(x => x.key_full.StartsWith(key_0 + "#") ||
x.key_full.StartsWith(key_0 + ".")).OrderBy(x => x.key_full).ToArray();
string[] keys_1 = okeys_1.Select(x => x.key_full.Split(new char[] { '#', '.' })[1]).Distinct().OrderBy(x => x).ToArray();
if (keys_1.Length == 0)
{
nodes_0[i].Tag = new oKey()
{
key0 = key_0, level = 0
};
}
else
{
for (var i1 = 0; i1 < keys_1.Length; i1++)
{
string key_1 = keys_1[i1];
var nodes_1 = new TreeNode(key_1);
nodes_0[i].Nodes.Add(nodes_1);
var okeys_2 = m_keys.Where(x => (x.key_full.StartsWith(key_0 + "#") || x.key_full.StartsWith(key_0 + ".")) &&
(x.key_full.Contains(key_1 + "#") || x.key_full.Contains(key_1 + "."))).OrderBy(x => x.key_full).ToArray();
string[] keys_2 = okeys_2.Select(x => x.key_full.Split(new char[] { '#', '.' })[2]).Distinct().OrderBy(x => x).ToArray();
if (keys_2.Length == 0)
{
nodes_1.Tag = new oKey()
{
key0 = key_0, key1 = key_1, level = 1
};
}
else
{
for (var i2 = 0; i2 < keys_2.Length; i2++)
{
string key_2 = keys_2[i2];
var nodes_2 = new TreeNode(key_2)
{
Tag = new oKey()
{
key0 = key_0, key1 = key_1, key2 = key_2, level = 2
}
};
nodes_1.Nodes.Add(nodes_2);
}
}
}
}
}
treeKeys.Nodes.AddRange(nodes_0);
//f_message("OK");
//Thread.Sleep(100);
treeKeys.ExpandAll();
//Thread.Sleep(100);
if (key_full_selected.Length > 0)
{
CallRecursive(key_full_selected);
}
}
catch (Exception err)
{
f_message("FAIL: " + err.Message);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RedisHashingAccessor" /> class.
/// </summary>
/// <param name="_redisDap">The initialized data access object we'll use to communicate.</param>
/// <param name="redisLogger">The logger that will be used in this class to create messages.</param>
public RedisHashingAccessor(RedisDataAccessProvider _redisDap,
ILogger redisLogger)
{
this.redisDap = _redisDap;
this.log = redisLogger;
}
/// <summary>
/// Executes a Redis command, when requested.
/// </summary>
/// <param name="redisDap">DAO that will communicate with the cache.</param>
/// <param name="log">Logger to output information about what's occurring.</param>
/// <param name="cmdToExecute">The command we're executing.</param>
/// <param name="paramArgs">The parameters to send with the command.</param>
public static void ExecuteRedisCommand(this RedisDataAccessProvider redisDap, ILogger log, RedisCommand cmdToExecute, params string[] paramArgs)
{
log.LogRedisMessage(cmdToExecute, paramArgs);
redisDap.SendCommand(cmdToExecute, paramArgs);
}
public RedisConnection()
{
redis = new RedisDataAccessProvider();
redis.Configuration.Host = IpAdress;
redis.Configuration.Port = Port;
}
