public void run() { try { string csdl = "interaction.type == \"twitter\" and interaction.sample <= 1.0"; m_form.deletesLog("Creating definition..."); m_form.deletesLog(" " + csdl); Definition def = m_form.user.createDefinition(csdl); m_form.deletesLog("Creating consumer..."); m_consumer = def.getConsumer(this, "http"); m_form.deletesLog("Consuming..."); m_consumer.consume(); } catch (Exception e) { m_form.deletesLog(e.GetType().ToString() + ": " + e.Message); if (m_consumer != null) { m_consumer.stop(); } } }
public void onDisconnect(StreamConsumer consumer) { m_form.footballLog("Disconnected"); m_form.footballLog("--"); m_form.resetFootballStartButton(); }
/// <summary> /// Saves the source stream to a unified file and adds a comment if the file implrments IVersioningFile. /// </summary> /// <param name="sourceStream">The source stream.</param> /// <param name="targetFile">The target file.</param> /// <param name="comment">A comment to add when checking in a versioning file.</param> public static void WriteToFile(Stream sourceStream, UnifiedFile targetFile, string comment) { // It may be a versioning file IVersioningFile versioningFile = targetFile as IVersioningFile; if (versioningFile != null) { ThrowIfCheckedOut(versioningFile); if (!versioningFile.IsCheckedOut) { versioningFile.CheckOut(); } } // Copy the source stream to the target file stream. using (Stream writeStream = targetFile.Open(FileMode.Create, FileAccess.Write)) { StreamConsumer.CopyToEnd(sourceStream, writeStream); } // If versioning, then check in with the supplied comment. if (versioningFile != null) { versioningFile.CheckIn(comment ?? String.Empty); } }
public virtual void Write(Stream stream) { if (_consumer == StreamConsumer.Reader) { throw new NotSupportedException("You cannot read & write to a BlockingStream from the same thread."); } _consumer = StreamConsumer.Writer; if (_dataAvailable == null) { throw new ObjectDisposedException(GetType().Name); } lock (_lockForAll) { if (_exception != null) { var ex = _exception; _exception = null; throw ex; } if (_illegalToWrite) { throw new InvalidOperationException( "Writing has already been completed."); } _chunks.Enqueue(stream); _dataAvailable.Set(); } }
public ExternalSolverSession(ProcessStartInfo startInfo) { _process = startInfo.Start(); Debug.Assert(_process != null); _stdin = _process.StandardInput.BaseStream; _stdout = _process.StandardOutput.BaseStream; _stderr = new StreamConsumer(_process.StandardError); var apiNotification = GetJsonChunk(); if (apiNotification == null || apiNotification[0].ToString() != "invoke" || apiNotification[1] != null || apiNotification[2].ToString() != "set-api-version") { throw new IOException("External solver did not respond correctly to handshake."); } var apiVersion = new ImplementationVersion(apiNotification[3].ReparseAsJson <string[]>()[0]); if (apiVersion >= new ImplementationVersion(ApiVersion)) { Log.Debug("Agreed on 0install slave API version " + apiVersion); } else { throw new IOException("Failed to agree on slave API version. External solver insisted on: " + apiVersion); } }
public static async ValueTask <byte[]> ToArrayAsync( this IStreamProducer producer, int copyBufferSize = StreamConsumer.DefaultBufferSize, CancellationToken cancellationToken = default) { await using var consumer = StreamConsumer.ToArray(copyBufferSize); return(await producer.ConsumeAsync(consumer, cancellationToken).ConfigureAwait(false)); }
public static async ValueTask <string> ToStringAsync( this IStreamProducer producer, Encoding?encoding = default, int copyBufferSize = StreamConsumer.DefaultBufferSize, CancellationToken cancellationToken = default) { await using var consumer = StreamConsumer.ToString(encoding, copyBufferSize); return(await producer.ConsumeAsync(consumer, cancellationToken).ConfigureAwait(false)); }
public IStreamConsumer CreateConsumer(ContentInfo contentInfo) => StreamConsumer.ToStream(new FileStream( AbsolutePath, FileMode.Create, FileAccess.Write, FileShare.ReadWrite, BufferSize ?? DefaultBufferSize, FileOptions.WriteThrough | FileOptions.Asynchronous ), BufferSize ?? DefaultBufferSize);
public void onInteraction(StreamConsumer consumer, Interaction interaction, string hash) { try { m_form.deletesLog(".", false); } catch (Exception e) { m_form.deletesLog(e.GetType().ToString() + ": " + e.Message); } }
public async Task Stream() { var seed = Encoding.ASCII.GetBytes(@"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce elementum nisi vel magna rhoncus, in aliquam ipsum accumsan. Phasellus efficitur lectus quis malesuada aliquet. Proin mattis sagittis magna vitae blandit. Cras vel diam sagittis, fringilla nunc vitae, vehicula mi. Nullam et auctor mi. Proin vel pharetra tortor. Donec posuere elementum risus, et aliquet magna pharetra non. Curabitur volutpat maximus sem at euismod. Fusce porta, lacus vel varius varius, lacus felis faucibus ante, fermentum sollicitudin elit neque rhoncus tortor. Aenean eget turpis consequat, luctus lorem vehicula, ullamcorper erat."); await using var prod = StreamProducer.FromStream(new MemoryStream(seed, false)); var buffer = new MemoryStream(); await using var cons = StreamConsumer.ToStream(buffer); await prod.ConsumeAsync(cons); Assert.True(seed.SequenceEqual(buffer.ToArray())); }
public void OnRead( Action <StreamHandle, ReadableBuffer> onAccept, Action <StreamHandle, Exception> onError, Action <StreamHandle> onCompleted = null) { Contract.Requires(onAccept != null); Contract.Requires(onError != null); var consumer = new StreamConsumer <StreamHandle>(onAccept, onError, onCompleted); this.pipeline.Consumer(consumer); }
/// <summary> /// Reads all currently pending <paramref name="stderr"/> lines and sends responses to <paramref name="stdin"/>. /// </summary> private void HandlePending([NotNull] StreamConsumer stderr, [NotNull] StreamWriter stdin) { string line; while ((line = stderr.ReadLine()) != null) { string response = HandleStderr(line); if (response != null) { stdin.WriteLine(response); } } }
public void onDeleted(StreamConsumer consumer, Interaction interaction, string hash) { try { m_form.footballLog("Delete request for interaction " + interaction.getStringVal("interaction.id") + " of type " + interaction.getStringVal("interaction.type")); m_form.footballLog("Please delete it from your archive."); m_form.footballLog("--"); } catch (Exception e) { m_form.footballLog(e.GetType().ToString() + ": " + e.Message); } }
protected virtual async Task <ImageInfo> InvokeGetImageInfoAsync( IImageSource source, string endpoint, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); using var scope = Logger.BeginScope(Guid.NewGuid()); Logger.LogDebug("Analyze operation starting."); var uri = new UriBuilder(endpoint).AppendPathSegment(Routes.Info).Uri; var context = await GetOperationContextAsync(source, endpoint, cancellationToken); Logger.LogDebug("Computed context for analyze operation ({0}).", context.ContentType); try { var consumer = StreamConsumer.Create((input, cancellationToken) => JsonSerializer.DeserializeAsync <ImageInfo>(input, _imageInfoOptions, cancellationToken)) .Chain(StreamTransformation.Create(async(input, output, cancellationToken) => { Logger.LogDebug("Sending analyze request."); using var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = new TypedStreamContent(input, context.ContentType) }; using var client = CreateHttpClient(); using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); Logger.LogDebug("Received response of the analyze request."); await CheckAsync(response, cancellationToken); using var stream = await response.Content.ReadAsStreamAsync(); await stream.CopyToAsync(output, 16 * 1024, cancellationToken); Logger.LogDebug("Done processing response of the analyze request."); })); Logger.LogDebug("Initializing analyze operation."); var result = await context.Producer.ConsumeAsync(consumer, cancellationToken).ConfigureAwait(false); Logger.LogDebug("Analyze operation completed."); return(result); } catch (Exception exn) when(!(exn is ImageException)) { if (IsSocketRelated(exn, out var socketExn)) { if (IsBrokenPipe(socketExn) && source.Reusable) { Logger.LogWarning(exn, "Failed to perform operation due to connection error, retrying..."); return(await InvokeGetImageInfoAsync(source, endpoint, cancellationToken)); } throw new RemoteImageConnectivityException(endpoint, socketExn.SocketErrorCode, "Network related error has occured while performing operation.", exn); } throw new RemoteImageException(endpoint, ErrorCodes.GenericError, "Error has occurred while performing operation.", exn); } }
public override void Write(byte[] buffer, int offset, int count) { if (_consumer == StreamConsumer.Reader) { throw new NotSupportedException("You cannot read & write to a BlockingStream from the same thread."); } _consumer = StreamConsumer.Writer; lock (_lockForAll) if (_exception != null) { var ex = _exception; _exception = null; throw ex; } if (buffer == null) { throw new ArgumentNullException("buffer"); } if (offset < 0 || offset >= buffer.Length) { throw new ArgumentOutOfRangeException("offset"); } if (count < 0 || offset + count > buffer.Length) { throw new ArgumentOutOfRangeException("count"); } if (_dataAvailable == null) { throw new ObjectDisposedException(GetType().Name); } if (count == 0) { return; } byte[] chunk = new byte[count]; Buffer.BlockCopy(buffer, offset, chunk, 0, count); lock (_lockForAll) { if (_illegalToWrite) { throw new InvalidOperationException( "Writing has already been completed."); } _chunks.Enqueue(chunk); _dataAvailable.Set(); } }
public void SetEndOfStream() { if (_consumer == StreamConsumer.Reader) { throw new NotSupportedException("You cannot read & write to a BlockingStream from the same thread."); } if (_dataAvailable == null) { throw new ObjectDisposedException(GetType().Name); } lock (_lockForAll) { _consumer = StreamConsumer.Undefined; _illegalToWrite = true; _doneWriting.Set(); } }
public void OnRead( Action <StreamHandle, ReadableBuffer> onAccept, Action <StreamHandle, Exception> onError, Action <StreamHandle> onCompleted = null) { if (onAccept is null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.onAccept); } if (onError is null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.onError); } var consumer = new StreamConsumer <StreamHandle>(onAccept, onError, onCompleted); _pipeline.Consumer(consumer); }
public void onDeleted(StreamConsumer consumer, Interaction interaction, string hash) { try { m_form.deletesLog("X", false); m_count--; if (m_count == 0) { m_form.deletesLog("\r\n\r\nStopping consumer..."); consumer.stop(); } } catch (Exception e) { m_form.deletesLog(e.GetType().ToString() + ": " + e.Message); } }
public void onInteraction(StreamConsumer consumer, Interaction interaction, string hash) { try { m_form.footballLog("Type: " + interaction.getStringVal("interaction.type")); m_form.footballLog("Content: " + interaction.getStringVal("interaction.content")); m_form.footballLog("--"); m_count--; if (m_count == 0) { m_form.footballLog("Stopping consumer..."); consumer.stop(); } } catch (Exception e) { m_form.footballLog(e.GetType().ToString() + ": " + e.Message); } }
public void run() { try { string csdl = "interaction.content contains \"football\""; m_form.footballLog("Creating definition..."); m_form.footballLog(" " + csdl); Definition def = m_form.user.createDefinition(csdl); m_form.footballLog("Creating consumer..."); m_consumer = def.getConsumer(this, "http"); m_form.footballLog("Consuming..."); m_consumer.consume(); } catch (Exception e) { m_form.footballLog(e.GetType().ToString() + ": " + e.Message); } }
public async Task EncryptAndDecryptAlternative() { var seed = @"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce elementum nisi vel magna rhoncus, in aliquam ipsum accumsan. Phasellus efficitur lectus quis malesuada aliquet. Proin mattis sagittis magna vitae blandit. Cras vel diam sagittis, fringilla nunc vitae, vehicula mi. Nullam et auctor mi. Proin vel pharetra tortor. Donec posuere elementum risus, et aliquet magna pharetra non. Curabitur volutpat maximus sem at euismod. Fusce porta, lacus vel varius varius, lacus felis faucibus ante, fermentum sollicitudin elit neque rhoncus tortor. Aenean eget turpis consequat, luctus lorem vehicula, ullamcorper erat."; var enc = new EncryptTransformation(); var dec = new DecryptTransformation(); var prod = StreamProducer.FromString(seed, Encoding.ASCII); var cons = StreamConsumer.ToString(Encoding.ASCII); var output = await prod.ConsumeAsync(cons .Chain(dec) .Chain(enc) ); Assert.Equal(seed, output); Assert.True(enc.HasStarted); Assert.True(enc.HasCompleted); Assert.Null(enc.Error); Assert.True(enc.HasBeenDisposed); Assert.True(dec.HasStarted); Assert.True(dec.HasCompleted); Assert.Null(dec.Error); Assert.True(dec.HasBeenDisposed); }
public override void Close() { if (_consumer == StreamConsumer.Reader) { _consumer = StreamConsumer.Undefined; base.Close(); if (_dataAvailable != null) { _dataAvailable.Close(); _dataAvailable = null; } if (_doneWriting != null) { _doneWriting.Close(); _doneWriting = null; } } else if (_consumer == StreamConsumer.Writer) { SetEndOfStream(); //_consumer = StreamConsumer.Undefined; } }
public virtual string Execute([NotNull] params string[] arguments) { #region Sanity checks if (arguments == null) { throw new ArgumentNullException(nameof(arguments)); } #endregion Process process; try { process = GetStartInfo(arguments).Start(); Debug.Assert(process != null); } #region Error handling catch (IOException ex) { throw new IOException(string.Format(Resources.UnableToLaunchBundled, AppBinary), ex); } #endregion var stdout = new StreamConsumer(process.StandardOutput); var stderr = new StreamConsumer(process.StandardError); var stdin = process.StandardInput; InitStdin(stdin); do { HandlePending(stderr, stdin); } while (!process.WaitForExit(50)); stdout.WaitForEnd(); stderr.WaitForEnd(); HandlePending(stderr, stdin); return(stdout.ToString()); }
public async Task CancelInsideAsyncProduce() { using var cancellationTokenSource = new CancellationTokenSource(); var data = new byte[128 * 1024]; var producer = StreamProducer.Create(async(output, _) => { while (true) { await output.WriteAsync(data.AsMemory(), cancellationTokenSource.Token); await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); } }); var consumerCancellationTriggered = false; var consumer = StreamConsumer.Create(async(input, cancellationToken) => { var buffer = new byte[64 * 1024]; try { while (true) { await input.ReadAsync(buffer.AsMemory(), cancellationToken); } } catch (OperationCanceledException) { consumerCancellationTriggered = true; throw; } }); var task = Assert.ThrowsAnyAsync <OperationCanceledException>(() => PipeStreamer.StreamAsync(producer, consumer, CancellationToken.None).AsTask()); cancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(50)); await task; Assert.True(consumerCancellationTriggered); }
public virtual void Read(Stream stream) { if (_consumer == StreamConsumer.Writer) { throw new NotSupportedException("You cannot read & write to a BlockingStream from the same thread."); } _consumer = StreamConsumer.Reader; if (_dataAvailable == null) { throw new ObjectDisposedException(GetType().Name); } lock (_lockForAll) if (_exception != null) { var ex = _exception; _exception = null; throw ex; } while (true) { int handleIndex = WaitHandle.WaitAny(_events); lock (_lockForRead) { lock (_lockForAll) { if (_currentChunk == null) { if (_chunks.Count == 0) { if (handleIndex == _doneWritingHandleIndex) { return; } else { continue; } } _currentChunk = _chunks.Dequeue(); _currentChunkPosition = 0; } } if (_currentChunk is Stream) { Streams.Copy((Stream)_currentChunk, stream); } else { var buffer = (byte[])_currentChunk; stream.Write(buffer, (int)_currentChunkPosition, buffer.Length); } _currentChunk = null; _currentChunkPosition = 0; lock (_lockForAll) { if (_chunks.Count == 0) { _dataAvailable.Reset(); } } } } }
private int RunNativeCommand(string command, string parameters, string workingDirectory, bool wait, bool consumeStreams) { string[] cmd; string[] args = new string[2]; args[0] = command; args[1] = parameters; int result = -1; try { //with this variable will be done the switching string osName = Environment.GetEnvironmentVariable("OS"); if (osName == null) { throw new JobExecutionException("Could not read environment variable for OS"); } if (osName.ToLower().IndexOf("windows") > -1) { cmd = new string[args.Length + 2]; cmd[0] = "cmd.exe"; cmd[1] = "/C"; for (int i = 0; i < args.Length; i++) { cmd[i + 2] = args[i]; } } else if (osName.ToLower().IndexOf("linux") > -1) { cmd = new string[3]; cmd[0] = "/bin/sh"; cmd[1] = "-c"; cmd[2] = args[0] + " " + args[1]; } else { // try this... cmd = args; } // Executes the command string temp = ""; for (int i = 1; i < cmd.Length; i++) { temp += cmd[i] + " "; } temp = temp.Trim(); Log.Info($"About to run {cmd[0]} {temp}..."); Process proc = new Process(); proc.StartInfo.FileName = cmd[0]; proc.StartInfo.Arguments = temp; #if WINDOWS_PROCESS proc.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; #endif // WINDOWS_PROCESS proc.StartInfo.CreateNoWindow = true; proc.StartInfo.UseShellExecute = false; proc.StartInfo.RedirectStandardError = true; proc.StartInfo.RedirectStandardOutput = true; if (!string.IsNullOrEmpty(workingDirectory)) { proc.StartInfo.WorkingDirectory = workingDirectory; } proc.Start(); // Consumes the stdout from the process StreamConsumer stdoutConsumer = new StreamConsumer(this, proc.StandardOutput.BaseStream, StreamTypeStandardOutput); // Consumes the stderr from the process if (consumeStreams) { StreamConsumer stderrConsumer = new StreamConsumer(this, proc.StandardError.BaseStream, StreamTypeError); stdoutConsumer.Start(); stderrConsumer.Start(); } if (wait) { proc.WaitForExit(); result = proc.ExitCode; } // any error message? } catch (Exception x) { throw new JobExecutionException("Error launching native command: " + x.Message, x, false); } return(result); }
public ValueTask PerformAsync(Stream input, Stream output, CancellationToken cancellationToken = default) => StreamAsync( producer: StreamProducer.Create((output, cancellationToken) => _first.PerformAsync(input, output, cancellationToken)), consumer: StreamConsumer.Create((input, cancellationToken) => _second.PerformAsync(input, output, cancellationToken)), cancellationToken: cancellationToken );
public ValueTask ProduceAsync(Stream output, CancellationToken cancellationToken = default) => StreamAsync( producer: _producer, consumer: StreamConsumer.Create((input, cancellationToken) => _transformation.PerformAsync(input, output, cancellationToken)), cancellationToken: cancellationToken );
public void onWarning(StreamConsumer consumer, string message) { m_form.footballLog("WARN: " + message); m_form.footballLog("--"); }
public override int Read(byte[] buffer, int offset, int count) { if (_consumer == StreamConsumer.Writer) { throw new NotSupportedException("You cannot read & write to a BlockingStream from the same thread."); } _consumer = StreamConsumer.Reader; lock (_lockForAll) if (_exception != null) { var ex = _exception; _exception = null; throw ex; } if (buffer == null) { throw new ArgumentNullException("buffer"); } if (offset < 0 || offset >= buffer.Length) { throw new ArgumentOutOfRangeException("offset"); } if (count < 0 || offset + count > buffer.Length) { throw new ArgumentOutOfRangeException("count"); } if (_dataAvailable == null) { throw new ObjectDisposedException(GetType().Name); } if (count == 0) { return(0); } while (true) { int handleIndex = WaitHandle.WaitAny(_events); lock (_lockForRead) { lock (_lockForAll) { if (_currentChunk == null) { if (_chunks.Count == 0) { if (handleIndex == _doneWritingHandleIndex) { return(0); } else { continue; } } _currentChunk = _chunks.Dequeue(); _currentChunkPosition = 0; } } if (_currentChunk is Stream) { var read = ((Stream)_currentChunk).Read(buffer, offset, count); if (read != count) { _currentChunk = null; _currentChunkPosition = 0; lock (_lockForAll) { if (_chunks.Count == 0) { _dataAvailable.Reset(); } } } else { _currentChunkPosition += count; } return(read); } else { long bytesAvailable = ((_currentChunk is Stream) ? ((Stream)_currentChunk).Length : ((byte[])_currentChunk).Length) - _currentChunkPosition; int bytesToCopy; if (bytesAvailable > count) { bytesToCopy = count; Buffer.BlockCopy((byte[])_currentChunk, (int)_currentChunkPosition, buffer, offset, count); _currentChunkPosition += count; } else { bytesToCopy = (int)bytesAvailable; Buffer.BlockCopy((byte[])_currentChunk, (int)_currentChunkPosition, buffer, offset, bytesToCopy); _currentChunk = null; _currentChunkPosition = 0; lock (_lockForAll) { if (_chunks.Count == 0) { _dataAvailable.Reset(); } } } return(bytesToCopy); } } } }
private int RunNativeCommand(string command, string parameters, string workingDirectory, bool wait, bool consumeStreams) { string[] cmd; string[] args = new string[2]; args[0] = command; args[1] = parameters; int result = -1; try { //with this variable will be done the switching string osName = Environment.GetEnvironmentVariable("OS"); if (osName == null) { throw new JobExecutionException("Could not read environment variable for OS"); } if (osName.ToLower().IndexOf("windows") > -1) { cmd = new string[args.Length + 2]; cmd[0] = "cmd.exe"; cmd[1] = "/C"; for (int i = 0; i < args.Length; i++) { cmd[i + 2] = args[i]; } } else if (osName.ToLower().IndexOf("linux") > -1) { cmd = new String[3]; cmd[0] = "/bin/sh"; cmd[1] = "-c"; cmd[2] = args[0] + " " + args[1]; } else { // try this... cmd = args; } // Executes the command string temp = ""; for (int i = 1; i < cmd.Length; i++) { temp += cmd[i] + " "; } temp = temp.Trim(); Log.Info(string.Format(CultureInfo.InvariantCulture, "About to run {0} {1}...", cmd[0], temp)); Process proc = new Process(); proc.StartInfo.FileName = cmd[0]; proc.StartInfo.Arguments = temp; proc.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; proc.StartInfo.CreateNoWindow = true; proc.StartInfo.UseShellExecute = false; proc.StartInfo.RedirectStandardError = true; proc.StartInfo.RedirectStandardOutput = true; if (!String.IsNullOrEmpty(workingDirectory)) { proc.StartInfo.WorkingDirectory = workingDirectory; } proc.Start(); // Consumes the stdout from the process StreamConsumer stdoutConsumer = new StreamConsumer(this, proc.StandardOutput.BaseStream, StreamTypeStandardOutput); // Consumes the stderr from the process if (consumeStreams) { StreamConsumer stderrConsumer = new StreamConsumer(this, proc.StandardError.BaseStream, StreamTypeError); stdoutConsumer.Start(); stderrConsumer.Start(); } if (wait) { proc.WaitForExit(); result = proc.ExitCode; } // any error message? } catch (Exception x) { throw new JobExecutionException("Error launching native command: " + x.Message, x, false); } return result; }