private void _ReconnectTimer(Nullable <Int32> reconnectMilliseconds)
{
try
{
lock (m_ReconnectLocker)
{
if (m_ReconnectTimer != null)
{
m_ReconnectTimer.Dispose();
m_ReconnectTimer = null;
}
// since a reconnect will definately occur we can stop the check timers for now until reconnect succeeds (where they are recreated)
_KillTimers();
LoggerQueue.Warn(">>>>>> REDIS STARTING RECONNECT TIMER >>>>>>");
m_ReconnectTimer = new Timer(_Connect, false, reconnectMilliseconds.GetValueOrDefault(m_ReconnectTime), Timeout.Infinite);
}
}
catch (Exception ex)
{
LoggerQueue.Error("Error during _ReconnectTimer", ex);
}
}
public LoggerWriter(string filename, LoggerQueue newLoggerQueue)
{
workingFile = filename;
logfile = new StreamWriter(workingFile, true);
loggerQueue = newLoggerQueue;
}
public Int64 Publish(string channel, RedisValue message)
{
try
{
return(_Redis.GetSubscriber().Publish(channel, message));
}
catch (RedisConnectionNotReadyException)
{
LoggerQueue.Error("REDIS RedisConnectionNotReadyException ERROR DURING Publish");
throw;
}
catch (RedisServerException ex)
{
LoggerQueue.Error("REDIS RedisServerException ERROR DURING Publish - " + ex.Message);
throw;
}
catch (RedisConnectionException ex)
{
LoggerQueue.Error("REDIS RedisConnectionException ERROR DURING Publish - " + ex.Message);
throw;
}
catch (TimeoutException ex)
{
LoggerQueue.Error("REDIS TimeoutException ERROR DURING Publish - " + ex.Message);
throw;
}
catch (Exception ex)
{
LoggerQueue.Error("REDIS Exception ERROR DURING Publish", ex);
throw;
}
}
private void _Execute(Action action)
{
try
{
action.Invoke();
}
catch (RedisServerException ex)
{
LoggerQueue.Error("REDIS RedisServerException ERROR DURING _Execute - " + ex.Message);
throw;
}
catch (RedisConnectionException ex)
{
LoggerQueue.Error("REDIS RedisConnectionException ERROR DURING _Execute - " + ex.Message);
throw;
}
catch (TimeoutException ex)
{
LoggerQueue.Error("REDIS TimeoutException ERROR DURING _Execute - " + ex.Message);
throw;
}
catch (Exception ex)
{
LoggerQueue.Error("REDIS Exception ERROR DURING _Execute", ex);
throw;
}
}
private TResult _Execute <TResult>(Func <TResult> function)
{
try
{
return(function.Invoke());
}
catch (RedisServerException ex)
{
LoggerQueue.Error("REDIS RedisServerException ERROR DURING _Execute - " + ex.Message);
throw;
}
catch (RedisConnectionException ex)
{
LoggerQueue.Error("REDIS RedisConnectionException ERROR DURING _Execute - " + ex.Message);
throw;
}
catch (TimeoutException ex)
{
LoggerQueue.Error("REDIS TimeoutException ERROR DURING _Execute - " + ex.Message);
throw;
}
catch (Exception ex)
{
LoggerQueue.Error("REDIS ERROR DURING _Execute", ex);
throw;
}
}
public void CreateTestRunWithFilename()
{
// Given
TestApiClient apiClient = new TestApiClient(_ => "{ \"id\": 1234 }");
LoggerQueue loggerQueue = new LoggerQueue(apiClient, "987", "foo", "bar")
{
Source = "Fizz.Buzz"
};
// When
int id = loggerQueue.CreateTestRun(CancellationToken.None).Result;
// Then
id.ShouldBe(1234);
apiClient.Messages.ShouldBe(new[]
{
new ClientMessage(
HttpMethod.Post,
null,
"5.0-preview.2",
$@"{{
""name"": ""Fizz.Buzz (OS: {System.Runtime.InteropServices.RuntimeInformation.OSDescription}, Job: bar, Agent: foo)"",
""build"": {{""id"":""987""}},
""startedDate"": ""{loggerQueue.StartedDate.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ")}"",
""isAutomated"": true
}}")
});
}
private void _KillRedisClient()
{
try
{
if (m_Redis != null)
{
Stopwatch sw = Stopwatch.StartNew();
LoggerQueue.Debug(">>>>>> KILLING REDIS CONNECTION >>>>>>");
ConnectionMultiplexer redis = m_Redis;
m_Redis = null;
if (this.IsSentinel == true)
{
redis.ConnectionFailed -= _ConnectionFailure;
redis.ConnectionRestored -= _ConnectionRestored;
}
redis.Close(false);
redis.Dispose();
LoggerQueue.Debug(">>>>>> KILLED REDIS CONNECTION >>>>>> " + sw.Elapsed);
}
}
catch (Exception ex)
{
LoggerQueue.Error(">>>>>> Error during _KillRedisClient", ex);
}
}
public void add_single_entry()
{
LoggerQueue queue = new LoggerQueue(20);
Assert.IsTrue(queue.add("Testing testing testing..."));
Assert.IsFalse(queue.add(null));
Assert.IsFalse(queue.add(""));
}
public void pop_single_entry()
{
LoggerQueue queue = new LoggerQueue(1);
queue.add("test");
Assert.IsTrue(queue.isFull());
Console.WriteLine(queue.pop());
Assert.IsTrue(queue.isEmpty());
}
public void GetMissingSource()
{
// Given
TestTestResult testResult = new TestTestResult();
// When
string source = LoggerQueue.GetSource(new[] { testResult });
// Then
source.ShouldBeNull();
}
void Start()
{
//if (ExperimentSettings_CoinTask.isLogging) {
myLoggerQueue = new LoggerQueue();
myLoggerWriter = new LoggerWriter(fileName, myLoggerQueue);
myLoggerWriter.Start();
myLoggerWriter.log("DATE: " + DateTime.Now.ToString("M/d/yyyy")); //might not be needed
//}
}
public void fill_queue()
{
LoggerQueue queue = new LoggerQueue(20);
for (int i = 0; i < queue.Size; i++)
{
queue.add("Testing testing testing...");
}
Assert.IsTrue(queue.isFull());
Assert.IsFalse(queue.add("Just one more..."));
}
void Start()
{
if (ExperimentSettings.isLogging)
{
myLoggerQueue = new LoggerQueue();
StartCoroutine("LogWriter");
// myLoggerWriter = new LoggerWriter (fileName, myLoggerQueue);
//
// myLoggerWriter.Start ();
// myLoggerWriter.log ("DATE: " + DateTime.Now.ToString ("M/d/yyyy")); //might not be needed
}
}
public void InitiateLogging()
{
if (Configuration.isLogging)
{
myLoggerQueue = new LoggerQueue();
StartCoroutine("LogWriter");
// myLoggerWriter = new LoggerWriter (fileName, myLoggerQueue);
//
// myLoggerWriter.Start ();
// myLoggerWriter.log ("DATE: " + DateTime.Now.ToString ("M/d/yyyy")); //might not be needed
}
}
public void GetSourceWithExtension()
{
// Given
TestTestResult testResult = new TestTestResult
{
Source = "/a/b/Foo.Bar.dll"
};
// When
string source = LoggerQueue.GetSource(new[] { testResult });
// Then
source.ShouldBe("Foo.Bar");
}
private void _OutputConfigurationFromSentinel()
{
m_SentinelServers.ForEach(s =>
{
try
{
IServer server = m_Sentinel.GetServer(s.Address, s.Port);
if (server.IsConnected == true)
{
try
{
IPEndPoint master = server.SentinelGetMasterAddressByName("master") as IPEndPoint;
var slaves = server.SentinelSlaves("master");
StringBuilder sb = new StringBuilder();
sb.Append(">>>>>> _OutputConfigurationFromSentinel Server ");
sb.Append(s.Address);
sb.Append(" thinks that master is ");
sb.Append(master);
sb.Append(" and slaves are ");
foreach (var slave in slaves)
{
string name = slave.Where(i => i.Key == "name").Single().Value;
bool up = slave.Where(i => i.Key == "flags").Single().Value.Contains("disconnected") == false;
sb.Append(name);
sb.Append("(");
sb.Append(up == true ? "connected" : "down");
sb.Append(") ");
}
sb.Append(">>>>>>");
LoggerQueue.Debug(sb.ToString());
}
catch (Exception ex)
{
LoggerQueue.Error(string.Format(">>>>>> _OutputConfigurationFromSentinel Could not get configuration from sentinel server ({0}) >>>>>>", s.Address), ex);
}
}
else
{
LoggerQueue.Error(string.Format(">>>>>> _OutputConfigurationFromSentinel Sentinel server {0} was not connected", s.Address));
}
}
catch (Exception ex)
{
LoggerQueue.Error(string.Format(">>>>>> _OutputConfigurationFromSentinel Could not get IServer from sentinel ({0}) >>>>>>", s.Address), ex);
}
});
}
private RedisConnection[] _GetAllRedisServersFromSentinel()
{
// ask each sentinel server for its configuration
List <RedisConnection> redisServers = new List <RedisConnection>();
m_SentinelServers.ForEach(s =>
{
try
{
IServer server = m_Sentinel.GetServer(s.Address, s.Port);
if (server.IsConnected == true)
{
try
{
// store master in list
IPEndPoint master = server.SentinelGetMasterAddressByName("master") as IPEndPoint;
redisServers.Add(new RedisConnection {
Address = master.Address.ToString(), Port = master.Port
});
var slaves = server.SentinelSlaves("master");
foreach (var slave in slaves)
{
string address = slave.Where(i => i.Key == "ip").Single().Value;
string port = slave.Where(i => i.Key == "port").Single().Value;
redisServers.Add(new RedisConnection {
Address = address, Port = Convert.ToInt32(port)
});
}
}
catch (Exception ex)
{
LoggerQueue.Error(string.Format(">>>>>> _GetAllRedisServersFromSentinel Could not get redis servers from sentinel server ({0}) >>>>>>", s.Address), ex);
}
}
else
{
LoggerQueue.Error(string.Format(">>>>>> _GetAllRedisServersFromSentinel Sentinel server {0} was not connected", s.Address));
}
}
catch (Exception ex)
{
LoggerQueue.Error(string.Format(">>>>>> _GetAllRedisServersFromSentinel Could not get IServer from sentinel ({0}) >>>>>>", s.Address), ex);
}
});
return(redisServers.Distinct().ToArray());
}
public void pop_multiple_entries()
{
LoggerQueue queue = new LoggerQueue(20);
int add = 0;
while (add < 10)
{
queue.add(String.Format("Test {0}", ++add));
}
int pop = 0;
while (pop < 10)
{
Assert.AreEqual(queue.pop(), String.Format("Test {0}", ++pop));
}
}
public void GroupTestResults()
{
// Given
TestApiClient apiClient = new TestApiClient();
LoggerQueue loggerQueue = new LoggerQueue(apiClient, "987", "foo", "bar")
{
Source = "Fizz.Buzz"
};
ITestResult[] testResults = new[]
{
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FooFixture.BarMethod"
},
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FooFixture.BazMethod"
},
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FutzFixture.BooMethod(\"x.y\")"
},
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FutzFixture.NestedFixture.BlitzMethod"
}
};
// When
IEnumerable <IGrouping <string, ITestResult> > testResultsByParent = loggerQueue.GroupTestResultsByParent(testResults);
// Then
testResultsByParent.Select(x => x.Key).ShouldBe(
new[]
{
"FooFixture",
"FutzFixture",
"FutzFixture.NestedFixture"
},
true);
}
private void _KillSentinelClient()
{
try
{
if (m_Sentinel != null)
{
LoggerQueue.Debug(">>>>>> KILLING SENTINEL CONNECTION >>>>>>");
ConnectionMultiplexer sentinel = m_Sentinel;
m_Sentinel = null;
sentinel.Close(false);
sentinel.Dispose();
}
}
catch (Exception ex)
{
LoggerQueue.Error(">>>>>> Error during _KillSentinelClient", ex);
}
}
private void _CheckSubscriptions(object state)
{
if (Monitor.TryEnter(m_ConnectionLocker, TimeSpan.FromSeconds(1)) == false)
{
return;
}
try
{
DateTime now = DateTime.UtcNow;
foreach (RedisSubscription subscription in m_Subscriptions.Values)
{
if ((now - subscription.LastUsed) > TimeSpan.FromSeconds(subscription.MaxNoReceiveSeconds))
{
try
{
EndPoint endpoint = m_Redis.GetSubscriber().IdentifyEndpoint(subscription.Channel);
EndPoint subscribedEndpoint = m_Redis.GetSubscriber().SubscribedEndpoint(subscription.Channel);
LoggerQueue.Warn(string.Format(">>>>>> REDIS Channel '{0}' has not been used for longer than {1}s, IsConnected: {2}, IsConnectedChannel: {3}, EndPoint: {4}, SubscribedEndPoint: {5}, reconnecting...", subscription.Channel, subscription.MaxNoReceiveSeconds, m_Redis.GetSubscriber().IsConnected(), m_Redis.GetSubscriber().IsConnected(subscription.Channel), endpoint != null ? endpoint.ToString() : "null", subscribedEndpoint != null ? subscribedEndpoint.ToString() : "null"));
}
catch (Exception ex)
{
LoggerQueue.Error(string.Format(">>>>>> REDIS Error logging out details of Channel '{0}' reconnect", subscription.Channel), ex);
}
_ReconnectTimer(null);
return;
}
}
}
catch (Exception ex)
{
LoggerQueue.Error(">>>>>> REDIS Exception ERROR during _CheckSubscriptions", ex);
}
finally
{
Monitor.Exit(m_ConnectionLocker);
}
}
private IServer _GetRedisMasterFromSentinel()
{
// ask each sentinel server for its configuration
foreach (RedisConnection sentinel in m_SentinelServers)
{
IServer sentinelServer = _Sentinel.GetServer(sentinel.Address, sentinel.Port);
if (sentinelServer.IsConnected == true)
{
try
{
IPEndPoint master = sentinelServer.SentinelGetMasterAddressByName("master") as IPEndPoint;
return(_Redis.GetServer(master));
}
catch (Exception ex)
{
LoggerQueue.Error(string.Format(">>>>>> Could not get redis master from sentinel server ({0}) >>>>>>", sentinel.Address), ex);
}
}
}
throw new InvalidOperationException("No sentinel server available to get master");
}
private void _CheckWrite(object state)
{
if (Monitor.TryEnter(m_ConnectionLocker, TimeSpan.FromSeconds(1)) == false)
{
return;
}
try
{
this.Database.HashSet(Environment.MachineName + "SmartoddsWriteCheck", m_CheckWriteGuid.ToString(), DateTime.UtcNow.Ticks);
}
catch (RedisConnectionNotReadyException)
{
LoggerQueue.Warn(">>>>>> REDIS RedisConnectionNotReadyException ERROR DURING _CheckWrite");
}
catch (RedisServerException ex)
{
LoggerQueue.Warn(">>>>>> REDIS RedisServerException ERROR DURING _CheckWrite, reconnecting... - " + ex.Message);
_ReconnectTimer(null);
}
catch (RedisConnectionException ex)
{
LoggerQueue.Warn(">>>>>> REDIS RedisConnectionException ERROR DURING _CheckWrite, reconnecting... - " + ex.Message);
_ReconnectTimer(null);
}
catch (TimeoutException ex)
{
LoggerQueue.Warn(">>>>>> REDIS TimeoutException ERROR DURING _CheckWrite - " + ex.Message);
}
catch (Exception ex)
{
LoggerQueue.Error(">>>>>> REDIS Exception ERROR during _CheckWrite", ex);
}
finally
{
Monitor.Exit(m_ConnectionLocker);
}
}
void Start ()
{
exp = GameObject.FindGameObjectWithTag ("Experiment").GetComponent<Experiment>();
if (!exp.isReplay) {
myLoggerQueue = new LoggerQueue ();
myLoggerWriter = new LoggerWriter (fileName, myLoggerQueue);
myLoggerWriter.Start ();
myLoggerWriter.log ("DATE: " + DateTime.Now.ToString ("M/d/yyyy")); //might not be needed
}
}
public LoggerWriter(string filename, LoggerQueue newLoggerQueue) {
workingFile = filename;
logfile = new StreamWriter ( workingFile, true );
loggerQueue = newLoggerQueue;
}
private void _Connect(object state)
{
bool throwException = (bool)state;
// if a reconnect is already being attempted, don't hang around waiting
if (Monitor.TryEnter(m_ConnectionLocker) == false)
{
return;
}
// we took the lock, notify everything we are connecting
m_Connecting = true;
try
{
Stopwatch sw = Stopwatch.StartNew();
LoggerQueue.Debug(">>>>>> REDIS CONNECTING... >>>>>>");
// if this is a reconnect, make absolutely sure everything is cleaned up first
_KillTimers();
_KillRedisClient();
if (this.IsSentinel == true && m_Sentinel == null)
{
LoggerQueue.Debug(">>>>>> CONNECTING TO SENTINEL >>>>>> - " + sw.Elapsed);
// we'll be getting the redis servers from sentinel
ConfigurationOptions sentinelConnection = _CreateRedisConfiguration(CommandMap.Sentinel, null, m_SentinelServers);
m_Sentinel = ConnectionMultiplexer.Connect(sentinelConnection);
LoggerQueue.Debug(">>>>>> CONNECTED TO SENTINEL >>>>>> - " + sw.Elapsed);
_OutputConfigurationFromSentinel();
// get all the redis servers from sentinel and ignore any set by caller
m_RedisServers.Clear();
m_RedisServers.AddRange(_GetAllRedisServersFromSentinel());
if (m_RedisServers.Count == 0)
{
throw new RedisException("Sentinel found no redis servers");
}
}
LoggerQueue.Debug(">>>>>> CONNECTING TO REDIS >>>>>> - " + sw.Elapsed);
// try to connect to all redis servers
ConfigurationOptions connection = _CreateRedisConfiguration(CommandMap.Default, _SecureStringToString(m_Password), m_RedisServers);
m_Redis = ConnectionMultiplexer.Connect(connection);
LoggerQueue.Debug(">>>>>> CONNECTED TO REDIS >>>>>> - " + sw.Elapsed);
// register subscription channels
m_Subscriptions.ForEach(s =>
{
m_Redis.GetSubscriber().Subscribe(s.Key, (channel, value) => _SubscriptionHandler(channel, value));
s.Value.LastUsed = DateTime.UtcNow;
});
if (this.IsSentinel == true)
{
// check subscriptions have been sending messages
if (m_Subscriptions.Count > 0)
{
m_CheckSubscriptionsTimer = new Timer(_CheckSubscriptions, null, 30000, m_CheckSubscriptionsTime);
}
if (m_CheckWriteTime != null)
{
// check that we can write to redis
m_CheckWriteTimer = new Timer(_CheckWrite, null, 32000, m_CheckWriteTime.Value);
}
// monitor for connection status change to any redis servers
m_Redis.ConnectionFailed += _ConnectionFailure;
m_Redis.ConnectionRestored += _ConnectionRestored;
}
LoggerQueue.Debug(string.Format(">>>>>> ALL REDIS CONNECTED ({0}) >>>>>>", sw.Elapsed));
}
catch (Exception ex)
{
LoggerQueue.Error(">>>>>> REDIS CONNECT FAILURE >>>>>>", ex);
if (throwException == true)
{
throw;
}
else
{
// internal reconnect, the reconnect has failed so might as well clean everything and try again
_KillTimers();
_KillRedisClient();
// faster than usual reconnect if failure
_ReconnectTimer(1000);
}
}
finally
{
// finished connection attempt, notify everything and remove lock
m_Connecting = false;
Monitor.Exit(m_ConnectionLocker);
}
}
public void CreateParents()
{
// Given
TestApiClient apiClient = new TestApiClient(_ =>
@"{
""count"": 2,
""value"": [
{
""id"": 100
},
{
""id"": 101
}
]
}");
LoggerQueue loggerQueue = new LoggerQueue(apiClient, "987", "foo", "bar")
{
Source = "Fizz.Buzz",
TestRunEndpoint = "/ep"
};
loggerQueue.Parents.Add("FitzFixture", new TestResultParent(123));
ITestResult[] testResults = new[]
{
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FooFixture.BarMethod"
},
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FitzFixture.BazMethod"
},
new TestTestResult
{
FullyQualifiedName = "Fizz.Buzz.FutzFixture.NestedFixture.BooMethod(\"x.y\")"
}
};
IEnumerable <IGrouping <string, ITestResult> > testResultsByParent = loggerQueue.GroupTestResultsByParent(testResults);
// When
loggerQueue.CreateParents(testResultsByParent, CancellationToken.None).Wait();
// Then
apiClient.Messages.ShouldBe(new[]
{
new ClientMessage(
HttpMethod.Post,
"/ep",
"5.0-preview.5",
$@"[
{{
""testCaseTitle"": ""FooFixture"",
""automatedTestName"": ""FooFixture"",
""resultGroupType"": ""generic"",
""outcome"": ""Passed"",
""state"": ""InProgress"",
""startedDate"": ""{loggerQueue.Parents["FooFixture"].StartedDate.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ")}"",
""automatedTestType"": ""UnitTest"",
""automatedTestTypeId"": ""13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"",
""automatedTestStorage"": ""Fizz.Buzz""
}},
{{
""testCaseTitle"": ""FutzFixture.NestedFixture"",
""automatedTestName"": ""FutzFixture.NestedFixture"",
""resultGroupType"": ""generic"",
""outcome"": ""Passed"",
""state"": ""InProgress"",
""startedDate"": ""{loggerQueue.Parents["FutzFixture.NestedFixture"].StartedDate.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ")}"",
""automatedTestType"": ""UnitTest"",
""automatedTestTypeId"": ""13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"",
""automatedTestStorage"": ""Fizz.Buzz""
}}
]")
});
loggerQueue.Parents.Keys.ShouldBe(new[] { "FitzFixture", "FooFixture", "FutzFixture.NestedFixture" }, true);
loggerQueue.Parents.Values.Select(x => x.Id).ShouldBe(new[] { 123, 100, 101 }, true);
}
private void _ConnectionRestored(object sender, ConnectionFailedEventArgs e)
{
LoggerQueue.Warn(string.Format(">>>>>> REDIS CONNECTION RESTORED, {0}, {1}, {2} >>>>>>", e.ConnectionType, e.EndPoint.ToString(), e.FailureType));
}
