public void Run(ISourceFunctionContext <string> ctx) { var attempt = 0; while (_isRunning) { _socket.RunAsync((message) => { // truncate trailing carriage return if (_delimiter.Equals("\n") && message.EndsWith("\r")) { message = message.Substring(0, message.Length - 1); } ctx.Collect(message); }, (ex) => { if (!_isRunning) { throw ex; } attempt++; if (_maxNumRetries == -1 || attempt < _maxNumRetries) { _logger.LogWarning("Lost connection to server socket. Retrying in " + _delayBetweenRetries + " msecs..."); Thread.Sleep(_delayBetweenRetries * attempt); } else { throw ex; } }, () => _isRunning = false).Wait(); } }
public void Run(ISourceFunctionContext <long> ctx) { while (_isRunning && _counter < 1000) { lock (ctx.GetCheckpointLock()) { ctx.Collect(_counter); _counter++; } } }
public void Run(ISourceFunctionContext <T> ctx) { var stream = new MemoryStream(_elementsSerialized); var input = new DataInputViewStreamWrapper(stream); // if we are restored from a checkpoint and need to skip elements, skip them now. var toSkip = _numElementsToSkip; if (toSkip <= 0) { return; } try { while (toSkip > 0) { _serializer.Deserialize(input); toSkip--; } } catch (Exception e) { throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + _serializer, e); } _numElementsEmitted = _numElementsToSkip; var mutex = ctx.GetCheckpointLock(); while (_isRunning && _numElementsEmitted < _numElements) { T next; try { next = _serializer.Deserialize(input); } catch (Exception e) { throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + _serializer, e); } lock (mutex) { ctx.Collect(next); _numElementsEmitted++; } } }
public override void Run(ISourceFunctionContext <Element> ctx) { var startTime = DateTime.Now.Ticks; var numElements = 20000000; var numKeys = 10000; var val = 1L; var count = 0L; while (_running && count < numElements) { count++; ctx.Collect(new Element(1L, val++)); if (val > numKeys) { val = 1L; } } var endTime = DateTime.Now.Ticks; Console.WriteLine("Took " + TimeSpan.FromTicks(endTime - startTime).TotalMilliseconds + " msecs for " + numElements + " values"); }