static void ScanDir(string dir, Channel<CrcResult> results, Channel<Exception> errors, WaitGroup wg) { foreach(var f in Directory.GetFiles(dir)) { var absPath = Path.Combine(dir, f); wg.Add(1); Go.Run(CalcCrc32, absPath, results, errors, wg); } foreach(var d in Directory.GetDirectories(dir)) { var absPath = Path.Combine(dir, d); wg.Add(1); Go.Run(ScanDir, absPath, results, errors, wg); } wg.Done(); }
public void Stop() { Trace.WriteLine("Stopping..."); // Stop all Consumers var wg = new WaitGroup(); foreach (var topicChannelHandler in _topicChannelHandlers) { foreach (var item in topicChannelHandler.Value) { var consumer = item.Consumer; if (consumer != null) { wg.Add(1); GoFunc.Run(() => { consumer.Stop(); wg.Done(); }, "NsqBus consumer shutdown"); } } } wg.Wait(); _nsqdPublisher.Stop(); Trace.WriteLine("Stopped."); }
private void BenchmarkTcp(int parallel) { string topicName = "test_benchmark_" + DateTime.Now.UnixNano(); try { const int benchmarkNum = 30000; byte[] body = new byte[512]; var p = new Producer("127.0.0.1:4150"); p.Connect(); var startCh = new Chan <bool>(); var wg = new WaitGroup(); for (int j = 0; j < parallel; j++) { wg.Add(1); GoFunc.Run(() => { startCh.Receive(); for (int i = 0; i < benchmarkNum / parallel; i++) { p.Publish(topicName, body); } wg.Done(); }, "ProducerBenchmarkTcpTest: sendLoop"); } var stopwatch = Stopwatch.StartNew(); startCh.Close(); var done = new Chan <bool>(); GoFunc.Run(() => { wg.Wait(); done.Send(true); }, "waiter and done sender"); bool finished = false; Select .CaseReceive(done, b => finished = b) .CaseReceive(Time.After(TimeSpan.FromSeconds(10)), b => finished = false) .NoDefault(); stopwatch.Stop(); if (!finished) { Assert.Fail("timeout"); } Console.WriteLine(string.Format("{0:#,0} sent in {1:mm\\:ss\\.fff}; Avg: {2:#,0} msgs/s; Threads: {3}", benchmarkNum, stopwatch.Elapsed, benchmarkNum / stopwatch.Elapsed.TotalSeconds, parallel)); p.Stop(); } finally { _nsqdHttpClient.DeleteTopic(topicName); _nsqLookupdHttpClient.DeleteTopic(topicName); } }
public static async Task Run() { var results = new Channel<CrcResult>(); var errors = new Channel<Exception>(); var wg = new WaitGroup(); wg.Add(1); Go.Run(async () => { // close the channels when the waitGroup signals await wg.Wait(); results.Close(); errors.Close(); }); Go.Run(ScanDir, "/Users/orion/OneDrive/Ignite2015/dev/goroutines", results, errors, wg); int totalFiles = 0; while(results.IsOpen || errors.IsOpen) { await Go.Select( Go.Case(results, r => { Console.WriteLine($"Got {r.Value} for {r.Path}"); totalFiles++; }), Go.Case(errors, exception => { Console.WriteLine($"EXCEPTION: {exception}"); totalFiles++; })); } Console.WriteLine($"{totalFiles} total files"); }
public void TwoSelectsSendAndReceiveCanTalk() { var c = new Chan <int>(); var actual = 0; var wg = new WaitGroup(); wg.Add(2); GoFunc.Run(() => { Select .CaseSend(c, 7) .NoDefault(); wg.Done(); }, "sender"); GoFunc.Run(() => { Select .CaseReceive(c, o => actual = o) .NoDefault(); wg.Done(); }, "receiver"); wg.Wait(); Assert.AreEqual(7, actual); }
static void ScanDir(string dir, Channel <CrcResult> results, Channel <Exception> errors, WaitGroup wg) { foreach (var f in Directory.GetFiles(dir)) { var absPath = Path.Combine(dir, f); wg.Add(1); Go.Run(CalcCrc32, absPath, results, errors, wg); } foreach (var d in Directory.GetDirectories(dir)) { var absPath = Path.Combine(dir, d); wg.Add(1); Go.Run(ScanDir, absPath, results, errors, wg); } wg.Done(); }
public static async Task Run() { var results = new Channel <CrcResult>(); var errors = new Channel <Exception>(); var wg = new WaitGroup(); wg.Add(1); Go.Run(async() => { // close the channels when the waitGroup signals await wg.Wait(); results.Close(); errors.Close(); }); Go.Run(ScanDir, "/Users/orion/OneDrive/Ignite2015/dev/goroutines", results, errors, wg); int totalFiles = 0; while (results.IsOpen || errors.IsOpen) { await Go.Select( Go.Case(results, r => { Console.WriteLine($"Got {r.Value} for {r.Path}"); totalFiles++; }), Go.Case(errors, exception => { Console.WriteLine($"EXCEPTION: {exception}"); totalFiles++; })); } Console.WriteLine($"{totalFiles} total files"); }
public void TestTcpConnWriteAfterClose() { var tcpListener = new TcpListener(IPAddress.Loopback, 4193); tcpListener.Start(); var wg = new WaitGroup(); wg.Add(1); GoFunc.Run(() => { var tcpClient = tcpListener.AcceptTcpClientAsync().Result; using (var rdr = new BinaryReader(tcpClient.GetStream())) using (var connw = new BinaryWriter(tcpClient.GetStream())) { while (true) { var readMsg = ReadBytes(rdr, (byte)'\n'); if (readMsg.SequenceEqual(Encoding.UTF8.GetBytes("QUIT\n"))) { break; } connw.Write(readMsg); } } tcpClient.Dispose(); wg.Done(); }, "TcpConnTest read loop"); var tcpConn = new TcpConn(IPAddress.Loopback.ToString(), 4193); var helloMsg = Encoding.UTF8.GetBytes("Hello\n"); tcpConn.Write(helloMsg, 0, helloMsg.Length); var recv = new byte[helloMsg.Length]; tcpConn.Read(recv); Console.WriteLine(Encoding.UTF8.GetString(recv)); var quitMsg = Encoding.UTF8.GetBytes("QUIT\n"); tcpConn.Write(quitMsg, 0, quitMsg.Length); recv = new byte[quitMsg.Length]; tcpConn.Read(recv); Console.WriteLine(Encoding.UTF8.GetString(recv)); wg.Wait(); tcpConn.Close(); AssertHelper.Throws <ConnectionClosedException>(() => tcpConn.Write(quitMsg, 0, quitMsg.Length)); }
public void BufferedChannelsSelectSendAndReceiveInGoroutine() { var c = new Chan <int>(10); var list = new List <int>(); var wg = new WaitGroup(); wg.Add(2); GoFunc.Run(() => { var doLoop = true; while (doLoop) { Select .CaseReceiveOk(c, (i, ok) => { if (ok) { list.Add(i); } else { doLoop = false; } }) .NoDefault(); } wg.Done(); }, "bufferChannelsTest:receiveLoop"); GoFunc.Run(() => { for (var i = 0; i < 10; i++) { Select .CaseSend(c, i) .NoDefault(); } c.Close(); wg.Done(); }, "bufferedChannelsTest:sendLoop"); wg.Wait(); AssertHelper.AreEqual(new[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, list); }
public void BufferedChannelsSelectSendInGoroutine() { var c = new Chan <int>(10); var list = new List <int>(); var wg = new WaitGroup(); wg.Add(1); GoFunc.Run(() => { for (int i = 0; i < 10; i++) { Select .CaseSend(c, i) .NoDefault(); } c.Close(); wg.Done(); }, "bufferedChannelsTest:sendLoop"); wg.Wait(); bool doLoop = true; // ReSharper disable once LoopVariableIsNeverChangedInsideLoop while (doLoop) { Select .CaseReceiveOk(c, (i, ok) => { if (ok) { list.Add(i); } else { doLoop = false; } }) .NoDefault(); } Assert.AreEqual(new[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, list.ToArray()); }
public void BufferedChannelsReceiveSelectInGoroutineSendOnMainThread() { var c = new Chan <int>(10); var list = new List <int>(); var wg = new WaitGroup(); wg.Add(1); GoFunc.Run(() => { bool doLoop = true; while (doLoop) { Select .CaseReceiveOk(c, (i, ok) => { if (ok) { list.Add(i); } else { doLoop = false; } }) .NoDefault(); } wg.Done(); }, "bufferChannelsTest:receiveLoop"); for (int i = 0; i < 10; i++) { c.Send(i); } c.Close(); wg.Wait(); Assert.AreEqual(new[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, list.ToArray()); }
private void BenchmarkHttp(int parallel) { string topicName = "test_benchmark_" + DateTime.Now.UnixNano(); try { const int benchmarkNum = 30000; byte[] body = new byte[512]; var startCh = new Chan <bool>(); var wg = new WaitGroup(); for (int j = 0; j < parallel; j++) { wg.Add(1); GoFunc.Run(() => { startCh.Receive(); for (int i = 0; i < benchmarkNum / parallel; i++) { _nsqdHttpClient.Publish(topicName, body); } wg.Done(); }, "ProducerBenchmarkHttpTest: sendLoop"); } var stopwatch = Stopwatch.StartNew(); startCh.Close(); wg.Wait(); stopwatch.Stop(); Console.WriteLine(string.Format("{0:#,0} sent in {1:mm\\:ss\\.fff}; Avg: {2:#,0} msgs/s; Threads: {3}", benchmarkNum, stopwatch.Elapsed, benchmarkNum / stopwatch.Elapsed.TotalSeconds, parallel)); } finally { _nsqdHttpClient.DeleteTopic(topicName); _nsqLookupdHttpClient.DeleteTopic(topicName); } }
public void TestSecureRandomGenerator() { var results = new ConcurrentBag <byte[]>(); var wg = new WaitGroup(); for (int i = 0; i < 10000; i++) { wg.Add(1); ThreadPool.QueueUserWorkItem(state => { results.Add(new SecureRandomGenerator().GenerateBytes(32)); wg.Done(); }); } wg.Await(); Assert.NotEmpty(results); var flag = results.Select(x => new EqualableBinary(x)) .GroupBy(x => x) .Any(x => x.Count() > 1); Assert.False(flag); }
public void Await() { var wg = new WaitGroup(); var i = new byte[] { 0 }; wg.Add(2); new Thread(new ParameterizedThreadStart((state) => { Thread.Sleep(100); i[0] = (byte)(i[0] + 1); wg.Done(); })).Start(); new Thread(new ParameterizedThreadStart((state) => { Thread.Sleep(100); i[0] = (byte)(i[0] + 1); wg.Done(); })).Start(); wg.Await(); Assert.AreNotEqual(i[0], 0); }
//[Test] public void TestConsoleLoggerThreadSafety() { var consoleLogger = new ConsoleLogger(LogLevel.Debug); var wg = new WaitGroup(); wg.Add(100); var rnd = new Random(); for (int i = 0; i < 100; i++) { int n = rnd.Next(100, 65536); var msg = new string('.', n); var t = new Thread(() => { consoleLogger.Output(LogLevel.Warning, msg); wg.Done(); }); t.IsBackground = true; t.Start(); } wg.Wait(); }
public void TestTimerStopRaceCondition() { // NOTE: This race condition was difficult to reproduce in Release but occurs // almost immediately in Debug. var wg = new WaitGroup(); var rand = new Random(); var passed = true; const int tries = 1000; wg.Add(tries); for (int i = 0; i < tries; i++) { GoFunc.Run(() => { try { var time = rand.Next(1, 2500); var timer = new Timer(TimeSpan.FromMilliseconds(time)); Time.AfterFunc(TimeSpan.FromMilliseconds(time), () => timer.Stop()); timer.C.Receive(); } catch (Exception ex) { Console.WriteLine(ex); passed = false; } wg.Done(); }, string.Format("timer {0}", i)); } wg.Wait(); Assert.IsTrue(passed); }
public void TestTickerStopRaceCondition() { // NOTE: This race condition was difficult to reproduce in Release but occurs // almost immediately in Debug. var wg = new WaitGroup(); var rand = new Random(); const int tries = 1000; wg.Add(tries); for (var i = 0; i < tries; i++) { var time = rand.Next(1, 500); var ticker = new Ticker(TimeSpan.FromMilliseconds(time)); Time.AfterFunc(TimeSpan.FromMilliseconds(time), () => { ticker.Close(); wg.Done(); }); } wg.Wait(); }
private void TestProducerReconnect(int publishingThreads, int millisecondsBetweenNsqdShutdown, int shutdownCount) { string topicName = string.Format("test_producerreconnect_{0}", DateTime.Now.UnixNano()); _nsqdHttpClient.CreateTopic(topicName); _nsqLookupdHttpClient.CreateTopic(topicName); try { var payload = new byte[512]; var publisher = new Producer("127.0.0.1:4150", new ConsoleLogger(LogLevel.Info), new Config()); bool running = true; for (int i = 0; i < publishingThreads; i++) { GoFunc.Run(() => { while (running) { try { publisher.PublishAsync(topicName, payload); } catch { } } }, string.Format("producer thread {0:00}/{1:00}", i + 1, publishingThreads)); } string errorMessage = null; var wg = new WaitGroup(); wg.Add(1); GoFunc.Run(() => { for (int i = 0; i < shutdownCount; i++) { Thread.Sleep(millisecondsBetweenNsqdShutdown); Console.WriteLine("Stopping nsqd {0}/{1}...", i + 1, shutdownCount); var p = new ProcessStartInfo("net", "stop nsqd"); p.CreateNoWindow = true; p.UseShellExecute = false; Process.Start(p).WaitForExit(); Console.WriteLine("Starting nsqd {0}/{1}...", i + 1, shutdownCount); p = new ProcessStartInfo("net", "start nsqd"); p.CreateNoWindow = true; p.UseShellExecute = false; Process.Start(p).WaitForExit(); Console.WriteLine("Attempting publish..."); // test the waters int tries; for (tries = 0; ; tries++) { try { publisher.Publish(topicName, payload); break; } catch (Exception ex) { Console.WriteLine(ex.Message); Thread.Sleep(1000); if (tries == 60) { errorMessage = string.Format("Producer not accepting Publish requests.\n" + "Producer Threads: {0}\nTime between NSQd shutdowns:{1}ms\n" + "Shutdown Count: {2}/{3}\nLast Exception Message: {4}", publishingThreads, millisecondsBetweenNsqdShutdown, i + 1, shutdownCount, ex.Message); Console.WriteLine(errorMessage); wg.Done(); return; } } } Console.WriteLine("Successful publish on attempt #{0}", tries + 1); } wg.Done(); }, "nsqd restart thread"); wg.Wait(); running = false; if (!string.IsNullOrEmpty(errorMessage)) { Assert.Fail(errorMessage); } Console.WriteLine("Starting test publishing of 1000 messages..."); for (int j = 0; j < 1000; j++) { publisher.Publish(topicName, payload); } Console.WriteLine("Done."); } finally { var p = new ProcessStartInfo("net", "start nsqd"); p.CreateNoWindow = true; p.UseShellExecute = false; Process.Start(p).WaitForExit(); _nsqdHttpClient.DeleteTopic(topicName); _nsqLookupdHttpClient.DeleteTopic(topicName); } }
private void RunTest(TestData td) { string topicName = string.Format("{0}_{1}", td.TopicPrefix, DateTime.Now.UnixNano()); string channelName = td.TopicPrefix; var container = new Container(); _nsqdHttpClient.CreateTopic(topicName); _nsqLookupdHttpClient.CreateTopic(topicName); var wg = new WaitGroup(); wg.Add(1); IFailedMessageInformation actualFailedMessageInfo = null; IMessageInformation actualSuccessMessageInfo = null; var fakeMessageAuditor = new Mock <IMessageAuditor>(MockBehavior.Strict); fakeMessageAuditor.Setup(p => p.OnReceived(It.IsAny <IBus>(), It.IsAny <IMessageInformation>())); fakeMessageAuditor.Setup(p => p.OnSucceeded(It.IsAny <IBus>(), It.IsAny <IMessageInformation>())) .Callback((IBus bus, IMessageInformation mi) => { if (actualSuccessMessageInfo != null) { throw new Exception("actualSuccessMessageInfo already set"); } actualSuccessMessageInfo = mi; wg.Done(); } ); fakeMessageAuditor.Setup(p => p.OnFailed(It.IsAny <IBus>(), It.IsAny <IFailedMessageInformation>())) .Callback((IBus bus, IFailedMessageInformation fmi) => { if (actualFailedMessageInfo != null) { throw new Exception("actualFailedMessageInfo already set"); } actualFailedMessageInfo = fmi; wg.Done(); } ); try { var nsqConfig = new Config { LookupdPollJitter = 0, LookupdPollInterval = TimeSpan.FromMilliseconds(10), DefaultRequeueDelay = TimeSpan.FromSeconds(90) }; if (td.MaxAttempts != null) { nsqConfig.MaxAttempts = td.MaxAttempts.Value; } var busConfig = new BusConfiguration( new StructureMapObjectBuilder(container), new NewtonsoftJsonSerializer(typeof(JsonConverter).Assembly), fakeMessageAuditor.Object, new MessageTypeToTopicDictionary(new Dictionary <Type, string> { { typeof(StubMessage), topicName } }), new HandlerTypeToChannelDictionary(new Dictionary <Type, string> { { td.HandlerType, channelName } }), defaultNsqLookupdHttpEndpoints: new[] { "127.0.0.1:4161" }, defaultThreadsPerHandler: 1, nsqConfig: nsqConfig, preCreateTopicsAndChannels: true ); BusService.Start(busConfig); var bus = container.GetInstance <IBus>(); // send the message and wait for the WaitGroup to finish. bus.Send(new StubMessage()); wg.Wait(); Thread.Sleep(200); // wait for nsqd to process the REQ if (td.ExpectOnSuccess) { Assert.IsNotNull(actualSuccessMessageInfo, "actualSuccessMessageInfo"); } else { Assert.IsNull(actualSuccessMessageInfo, "actualSuccessMessageInfo"); } if (td.ExpectOnFailed) { Assert.IsNotNull(actualFailedMessageInfo, "actualFailedMessageInfo"); Assert.AreEqual(td.FailedMessageReason, actualFailedMessageInfo.FailedReason, "failedReason"); Assert.AreEqual(td.FailedMessageQueueAction, actualFailedMessageInfo.FailedAction, "failedAction"); } else { Assert.IsNull(actualFailedMessageInfo, "actualFailedMessageInfo"); } // checks stats from http server var stats = _nsqdHttpClient.GetStats(); var topic = stats.Topics.Single(p => p.TopicName == topicName); var channel = topic.Channels.Single(p => p.ChannelName == channelName); Assert.AreEqual(1, topic.MessageCount, "topic.MessageCount"); Assert.AreEqual(0, topic.Depth, "topic.Depth"); Assert.AreEqual(0, topic.BackendDepth, "topic.BackendDepth"); Assert.AreEqual(1, channel.MessageCount, "channel.MessageCount"); // note: until the Requeue Timeout elapses the message is considered Deferred Assert.AreEqual(td.RequeueCount, channel.DeferredCount, "channel.DeferredCount"); Assert.AreEqual(0, channel.Depth, "channel.Depth"); Assert.AreEqual(0, channel.BackendDepth, "channel.BackendDepth"); Assert.AreEqual(0, channel.InFlightCount, "channel.InFlightCount"); Assert.AreEqual(0, channel.TimeoutCount, "channel.TimeoutCount"); Assert.AreEqual(0, channel.RequeueCount, "channel.RequeueCount"); } finally { BusService.Stop(); _nsqdHttpClient.DeleteTopic(topicName); _nsqLookupdHttpClient.DeleteTopic(topicName); } }
public void StartBus() { // TODO: Needs to move to NsqBus. See below comment about async bus start. // TODO: This also makes an assumption nsqd is running locally on port 4151. Convenient for testing and sample // TODO: apps, probably shouldn't be used in PROD. This needs to be thought through. if (_preCreateTopicsAndChannels) { const string nsqdHttpAddress = "127.0.0.1:4151"; var nsqdHttpClient = new NsqdHttpClient(nsqdHttpAddress, TimeSpan.FromSeconds(5)); var wg = new WaitGroup(); foreach (var tch in GetHandledTopics()) { foreach (var channel in tch.Channels) { string localTopic = tch.Topic; string localChannel = channel; wg.Add(1); GoFunc.Run(() => { try { nsqdHttpClient.CreateTopic(localTopic); nsqdHttpClient.CreateChannel(localTopic, localChannel); } catch (Exception ex) { _nsqLogger.Output(LogLevel.Error, string.Format("error creating topic/channel on {0} - {1}", nsqdHttpAddress, ex)); } wg.Done(); }, "BusConfiguration pre-create topics/channels"); } } wg.Wait(); } if (_busStateChangedHandler != null) { _busStateChangedHandler.OnBusStarting(this); } _bus = new NsqBus( _topicChannelHandlers, _dependencyInjectionContainer, _messageTypeToTopicProvider, _defaultMessageSerializer, _nsqLogger, _messageMutator, _messageTopicRouter, _nsqdPublisher ); _bus.Start(); // TODO: BusConfiguration should not be responsible for these callbacks. With an async _bus.Start // TODO: this will need to be moved to NsqBus. if (_busStateChangedHandler != null) { _busStateChangedHandler.OnBusStarted(this, _bus); } }
private void transactionCleanup() { // clean up transactions we can easily account for var wg = new WaitGroup(); wg.Add(_transactions.Count); foreach (var t in _transactions) { var t1 = t; GoFunc.Run(() => { t1.Error = new ErrNotConnected(); t1.finish(); wg.Done(); }, "transactionCleanup: drain _transactions"); } _transactions.Clear(); // spin and free up any writes that might have raced // with the cleanup process (blocked on writing // to transactionChan) // give the runtime a chance to schedule other racing goroutines var ticker = new Ticker(TimeSpan.FromMilliseconds(100)); bool doLoop = true; using (var select = Select .CaseReceive(_transactionChan, t => { wg.Add(1); GoFunc.Run(() => { t.Error = new ErrNotConnected(); t.finish(); wg.Done(); }, "transactionCleanup: finish transaction from _transactionChan"); }) .CaseReceive(ticker.C, _ => { // keep spinning until there are 0 concurrent producers if (_concurrentProducers == 0) { doLoop = false; return; } log(LogLevel.Warning, string.Format( "waiting for {0} concurrent producers to finish", _concurrentProducers)); }) .NoDefault(defer: true) ) { while (doLoop) { select.Execute(); } } ticker.Close(); wg.Wait(); }
/// <summary> /// Connect dials and bootstraps the nsqd connection /// (including IDENTIFY) and returns the IdentifyResponse /// </summary> public IdentifyResponse Connect() { var conn = Net.DialTimeout("tcp", _addr, _config.DialTimeout); _conn = (ITcpConn)conn; if (_conn == null) { throw new Exception("Net.DialTimeout returned null"); } _r = conn; _w = conn; _conn.ReadTimeout = _config.ReadTimeout; _conn.WriteTimeout = _config.WriteTimeout; try { Write(Protocol.MagicV2, 0, Protocol.MagicV2.Length); } catch (Exception ex) { _conn.Close(); throw new Exception(string.Format("[{0}] failed to write magic - {1}", _addr, ex.Message), ex); } IdentifyResponse resp; try { resp = identify(); } catch (ErrIdentify ex) { if (_addr.Contains(":4151")) { throw new ErrIdentify("Error connecting to nsqd. It looks like you tried to connect to the HTTP port " + "(4151), use the TCP port (4150) instead.", ex); } else if (_addr.Contains(":4160") || _addr.Contains(":4161")) { throw new ErrIdentify("Error connecting to nsqd. It looks like you tried to connect to nsqlookupd. " + "Producers must connect to nsqd over TCP (4150). Consumers can connect to nsqd over TCP (4150) using " + "Consumer.ConnectToNsqd or to nsqlookupd (4161) using Consumer.ConnectToNsqLookupd.", ex); } throw; } if (resp != null && resp.AuthRequired) { if (string.IsNullOrEmpty(_config.AuthSecret)) { log(LogLevel.Error, "Auth Required"); throw new Exception("Auth Required"); } auth(_config.AuthSecret); } _wg.Add(2); _readLoopRunning = 1; GoFunc.Run(readLoop, "Conn:readLoop"); GoFunc.Run(writeLoop, "Conn:writeLoop"); return(resp); }