static void Main(string[] args) { //使用Rx Observable对象封装async代码 //将Task<T>转换为IObservable<T> var client = new HttpClient(); IObservable <HttpResponseMessage> response = client.GetAsync("http://kworb.net/pop/").ToObservable(); //另一种方法是调用StartAsync IObservable <HttpResponseMessage> response2 = Observable.StartAsync(token => client.GetAsync("http://kworb.net/pop/", token)); //这两个方法都会立即启动异步操作,如果想要其接受订阅后启动,可使用FromAsync IObservable <HttpResponseMessage> response3 = Observable.FromAsync(token => client.GetAsync("http://kworb.net/pop/", token)); //Rx Observable对象和数据流网络 var buffer = new BufferBlock <int>(); IObservable <int> integers = buffer.AsObservable(); //创建一个缓冲块到Observable对象的接口 integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("done")); //如果使用一个网格并把它作为观察流的目的,情况会稍微复杂一点。 //ticks->display->done IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)).Timestamp().Select(x => x.Timestamp).Take(5); var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); try { display.Completion.Wait(); Console.WriteLine("Done."); }catch (Exception ex) { Console.WriteLine(ex); } }
private static void TdfRxGroupBy() { var buffer = new BufferBlock <int>(); Task.Run(() => { for (int i = 0; i < 25; i++) { buffer.Post(i); Thread.Sleep(500); } }); Console.WriteLine("Start"); Thread.Sleep(3000); var groups = from item in buffer.AsObservable() group item by item % 5 into g select g; groups.Subscribe(g => { ConsoleColor c = (ConsoleColor)(g.Key + 10); var ab = new ActionBlock <int>(item => { lock (groups) { Console.ForegroundColor = c; Console.Write("{0},", item); } }); g.Subscribe(ab.AsObserver()); }); }
static void Main(string[] args) { // 考虑把数据流块用作可观察流的输入 var buffer = new BufferBlock <int>(); IObservable <int> integers = buffer.AsObservable(); integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("done")); buffer.Post(13); buffer.Complete(); buffer.Completion.Wait(); // 考虑使用数据流网络并把它当作可观察流的终点 IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)) .Timestamp() .Select(x => x.Timestamp) .Take(5); var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); try { display.Completion.Wait(); Console.WriteLine("Done"); } catch (Exception ex) { Console.WriteLine(ex); } }
public static Task ExecuteInGroupsAsync <T>( this IEnumerable <T> source, Func <T, Task> func, int groupSize) { var block = new ActionBlock <IEnumerable <T> >( g => Task.WhenAll(g.Select(func))); source.ToObservable() .Buffer(groupSize) .Subscribe(block.AsObserver()); return(block.Completion); }
public ElasticSearchOutput(ElasticSearchOutputConfig config, PutIndexTemplateDescriptor indexTemplate = null) { _logger = LogProvider.For <ElasticSearchOutput>(); _logger.Info("Creating"); var connectionSettings = new ConnectionSettings(new Uri(config.ElasticSearchUri)) .RequestTimeout(config.RetryInterval) .MaximumRetries(config.RetryCount); _elasticClient = new ElasticClient(connectionSettings); var batchCount = config.MaxBatchCount > 0 ? config.MaxBatchCount : 1; _uploadBlock = new ActionBlock <IList <TelemetryEvent> >(list => Send(list, throwExceptions: false), new ExecutionDataflowBlockOptions { BoundedCapacity = config.BoundedCapacity, MaxDegreeOfParallelism = config.MaxDegreeOfParallelism, }); _internalStreamBufferSubscription = _internalStream .Buffer(TimeSpan.FromSeconds(config.BufferTimeSec), batchCount) .Subscribe(_uploadBlock.AsObserver()); _jsonSerializerSettings = new JsonSerializerSettings() { ContractResolver = new CamelCasePropertyNamesContractResolver() }; _elasticClient.PutIndexTemplate(indexTemplate ?? new PutIndexTemplateDescriptor(config.ElasticSearchIndexName + "-template") .Template(config.ElasticSearchIndexName + "*") .Settings(x => x .NumberOfReplicas(0) .NumberOfShards(1)) .Mappings(m => m .Map("_default_", tm => tm .AllField(af => af.Enabled(false)) .DynamicTemplates(d => d .DynamicTemplate("all_strings_not_analyzed", dd => dd .MatchMappingType("string") .Match("*") .Mapping(dm => dm .String(sm => sm .NotAnalyzed())))) .Properties(p => p .String(sp => sp .Name("message") .Index(FieldIndexOption.Analyzed)))))); _config = config; }
public void ExampleObservableToActionBlock() { IObservable <long> ticks = Observable.Interval(TimeSpan.FromMilliseconds(1)) .Select(t => t) .Take(5); var block = new ActionBlock <long>(x => Console.WriteLine($"ActionBlock: {x}")); ticks.Subscribe(block.AsObserver()); block.Completion.Wait(); }
public ElasticSearchOutput(ElasticSearchOutputConfig config, PutIndexTemplateDescriptor indexTemplate = null) { _logger = LogProvider.For<ElasticSearchOutput>(); _logger.Info("Creating"); var connectionSettings = new ConnectionSettings(new Uri(config.ElasticSearchUri)) .RequestTimeout(config.RetryInterval) .MaximumRetries(config.RetryCount); _elasticClient = new ElasticClient(connectionSettings); var batchCount = config.MaxBatchCount > 0 ? config.MaxBatchCount : 1; _uploadBlock = new ActionBlock<IList<TelemetryEvent>>(list => Send(list, throwExceptions: false), new ExecutionDataflowBlockOptions { BoundedCapacity = config.BoundedCapacity, MaxDegreeOfParallelism = config.MaxDegreeOfParallelism, }); _internalStreamBufferSubscription = _internalStream .Buffer(TimeSpan.FromSeconds(config.BufferTimeSec), batchCount) .Subscribe(_uploadBlock.AsObserver()); _jsonSerializerSettings = new JsonSerializerSettings() { ContractResolver = new CamelCasePropertyNamesContractResolver() }; _elasticClient.PutIndexTemplate(indexTemplate ?? new PutIndexTemplateDescriptor(config.ElasticSearchIndexName+"-template") .Template(config.ElasticSearchIndexName+"*") .Settings(x => x .NumberOfReplicas(0) .NumberOfShards(1)) .Mappings(m => m .Map("_default_", tm => tm .AllField(af => af.Enabled(false)) .DynamicTemplates(d => d .DynamicTemplate("all_strings_not_analyzed", dd => dd .MatchMappingType("string") .Match("*") .Mapping(dm => dm .String(sm => sm .NotAnalyzed())))) .Properties(p => p .String(sp => sp .Name("message") .Index(FieldIndexOption.Analyzed)))))); _config = config; }
//调用AsObserver让一个块订阅一个可观察流。 public static void ObservableToBlock() { IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)).Timestamp().Select(x => x.Timestamp).Take(5); var display = new ActionBlock <DateTimeOffset>(x => Trace.WriteLine(x)); ticks.Subscribe(display.AsObserver()); try { display.Completion.Wait(); Trace.WriteLine("Dowe."); } catch (Exception ex) { Trace.WriteLine(ex); } }
public static void UseToObservable() { var ob = T().ToObservable(); var ob2 = Observable.FromAsync(T); var buffer = new BufferBlock <int>(); var integers = buffer.AsObservable(); integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("Done")); buffer.Post(12); var client = new HttpClient(); var response = client.GetAsync("http://www.baidu.com").ToObservable(); var response2 = Observable.StartAsync(token => client.GetAsync("http://www.baidu.com", token)); IObservable <string> urls = new List <string>() { "http://www.baidu.coM" }.ToObservable(); var response3 = urls.SelectMany((url, token) => client.GetAsync(url, token)); var ticks = Observable.Interval(TimeSpan.FromSeconds(1)) .Timestamp() .Select(x => x.Timestamp) .Take(5); var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); try { display.Completion.Wait(); Console.WriteLine("Done"); } catch (Exception e) { Console.WriteLine(e); } }
public async Task TestAsObservableAndAsObserver_DataPropagation() { // Test that preset data flows correctly { var bb = new BufferBlock<int>(); bb.PostRange(0, 2); bb.Complete(); int nextValueExpected = 0; var ab = new ActionBlock<int>(i => { Assert.True(i == nextValueExpected, string.Format("Expected next value to be {0} but got {1}", nextValueExpected, i)); nextValueExpected++; }); bb.AsObservable().Subscribe(ab.AsObserver()); await ab.Completion; } // Test that new data flows correctly { int nextValueExpected = -2; var ab = new ActionBlock<int>(i => { Assert.True(i == nextValueExpected, string.Format("Expected next value to be {0} but got {1}", nextValueExpected, i)); nextValueExpected++; }); var bb = new BufferBlock<int>(); bb.AsObservable().Subscribe(ab.AsObserver()); bb.PostRange(-2, 0); bb.Complete(); await ab.Completion; } // Test that unsubscribing stops flow of data and stops completion { var target = new BufferBlock<int>(); var source = new BufferBlock<int>(); using (source.AsObservable().Subscribe(target.AsObserver())) { source.PostItems(1, 2); Assert.Equal(expected: 1, actual: await target.ReceiveAsync()); Assert.Equal(expected: 2, actual: await target.ReceiveAsync()); } source.Post(3); var wb = new WriteOnceBlock<int>(i => i); source.LinkTo(wb); await wb.Completion; source.Complete(); await source.Completion; Assert.False(target.Completion.IsCompleted); } }
public static void Main(string[] args) { var testObs = new TransformBlock <int, int>(item => item * item); var o = testObs.AsObservable(); o.Subscribe(i => Console.WriteLine("Test Obs received {0}", i.ToString())); for (int i = 0; i < 5; i++) { testObs.Post(i); } testObs.Completion.Wait(); Console.ReadKey(); var buffer = new BufferBlock <int>(); IObservable <int> integers = buffer.AsObservable(); integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("Done")); buffer.Post(13); buffer.Post(14); buffer.Post(15); buffer.Post(16); Console.ReadKey(); IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)) .Timestamp() .Select(x => x.Timestamp) .Take(5); var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); Console.ReadKey(); try { display.Completion.Wait(); Trace.WriteLine("Done."); } catch (Exception ex) { Trace.WriteLine(ex); } try { var multiplyBlock = new TransformBlock <int, int>(item => { // if (item == 1) // throw new InvalidOperationException("Blech."); return(item * 2); }); var subtractBlock = new TransformBlock <int, int>(item => item - 2); multiplyBlock.LinkTo(subtractBlock); var printing = new ActionBlock <int>(i => Console.WriteLine("Received {0}", i.ToString())); subtractBlock.LinkTo(printing, new DataflowLinkOptions { PropagateCompletion = true }); IObservable <int> obs = subtractBlock.AsObservable(); obs.Subscribe(i => Console.WriteLine("Received Observable {0}", i.ToString())); for (int i = 0; i < 10; i++) { multiplyBlock.Post(i); subtractBlock.Post(i); } Console.ReadLine(); //printing.Completion.Wait(); } catch (AggregateException exception) { AggregateException ex = exception.Flatten(); System.Diagnostics.Trace.WriteLine(ex.InnerException); } Console.WriteLine("Hello World!"); Console.ReadLine(); }
public static async Task Download(CloudStorageAccount storageAccount, DateTime startTimeInclusive, DateTime endTimeExclusive, StreamWriter writer, string cacheDirectory = null) { var telemetry = new TelemetryClient(); var stopwatch = Stopwatch.StartNew(); var blobClient = storageAccount.CreateCloudBlobClient(); var blobsToDownload = new List <DownloadItem>(); telemetry.TrackEvent($"Downloading {startTimeInclusive:yyyy-MM-dd} to {endTimeExclusive:yyyy-MM-dd}"); long downloadedBytes = 0; var options = new BlobRequestOptions { MaximumExecutionTime = new TimeSpan(0, 60, 0), ServerTimeout = new TimeSpan(0, 60, 0), RetryPolicy = new ExponentialRetry() }; var downloadBlock = new ActionBlock <DownloadItem>(async item => { try { if (cacheDirectory != null) { Console.WriteLine($"Downloading {item.Blob.Uri}"); } var temp = item.Filename + ".temp"; await item.Blob.DownloadToFileAsync(temp, FileMode.Create, AccessCondition.GenerateEmptyCondition(), options, new OperationContext()) .ContinueWith(t => File.Move(temp, item.Filename)); Interlocked.Add(ref downloadedBytes, item.Blob.Properties.Length); } catch (Exception e) { throw new Exception("Failed to download: " + item.Blob.Uri, e); } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 16, MaxDegreeOfParallelism = 8 }); var downloadInput = downloadBlock.AsObserver(); // enumerate full days for (var currentDateTime = new DateTime(startTimeInclusive.Year, startTimeInclusive.Month, startTimeInclusive.Day); currentDateTime < endTimeExclusive; currentDateTime += TimeSpan.FromDays(1)) { foreach (var blob in blobClient.ListBlobs($"joined-examples/{currentDateTime:yyyy/MM/dd}", useFlatBlobListing: true).OfType <CloudBlockBlob>()) { // find relevant folder var match = Regex.Match(blob.Uri.AbsolutePath, @"/(?<hour>\d{2})/(?<name>[^/]+\.json)$"); if (!match.Success) { telemetry.TrackTrace($"Skipping nvalid blob '{blob.Uri}'."); continue; } var blobDateTime = currentDateTime + TimeSpan.FromHours(int.Parse(match.Groups["hour"].Value, CultureInfo.InvariantCulture)); if (!(blobDateTime >= startTimeInclusive && blobDateTime < endTimeExclusive)) { continue; } var item = new DownloadItem { Blob = blob, DateTime = blobDateTime }; blobsToDownload.Add(item); if (cacheDirectory != null) { Console.WriteLine($"Checking {currentDateTime:yyyy/MM/dd}{match.Value}..."); // create directory var outputDir = Path.Combine(cacheDirectory, $"{blobDateTime:yyyy/MM/dd/HH}"); if (!Directory.Exists(outputDir)) { Directory.CreateDirectory(outputDir); } var outputFile = Path.Combine(outputDir, match.Groups["name"].Value); item.Filename = outputFile; if (!File.Exists(item.Filename)) { downloadInput.OnNext(item); } } } } downloadInput.OnCompleted(); await downloadBlock.Completion; if (cacheDirectory != null) { Console.WriteLine($"Download time: {stopwatch.Elapsed}"); } foreach (var blobGroup in blobsToDownload.GroupBy(i => i.DateTime).OrderBy(i => i.Key)) { if (cacheDirectory != null) { Console.WriteLine($"Merging {blobGroup.Key:yyyy/MM/dd HH}"); } // TODO: only use blob var dsBlobs = blobGroup.Select(b => new DecisionServiceBlob(b)).ToArray(); // not really faster and more error prone //var priorityQueue = new FastPriorityQueue<DecisionServiceEvent>(dsBlobs.Length); var priorityQueue = new SimplePriorityQueue <DecisionServiceEvent>(); foreach (var di in dsBlobs) { var evt = await di.Receive(); if (evt != null) { evt.Blob = di; priorityQueue.Enqueue(evt, evt.DateTime.Ticks); } } while (true) { var top = priorityQueue.First; // add VW string format writer.WriteLine(top.Line); if (await top.Next()) { priorityQueue.UpdatePriority(top, top.DateTime.Ticks); } else { priorityQueue.Dequeue(); if (priorityQueue.Count == 0) { break; } } } } if (cacheDirectory != null) { Console.WriteLine($"Total time: {stopwatch.Elapsed}"); } }
public static void Main(string[] args) { var testObs = new TransformBlock<int, int>(item => item * item); var o = testObs.AsObservable (); o.Subscribe(i=> Console.WriteLine("Test Obs received {0}", i.ToString())); for (int i = 0; i < 5; i++) { testObs.Post (i); } testObs.Completion.Wait (); Console.ReadKey (); var buffer = new BufferBlock<int>(); IObservable<int> integers = buffer.AsObservable(); integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("Done")); buffer.Post(13); buffer.Post(14); buffer.Post(15); buffer.Post(16); Console.ReadKey (); IObservable<DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)) .Timestamp() .Select(x => x.Timestamp) .Take(5); var display = new ActionBlock<DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); Console.ReadKey (); try { display.Completion.Wait(); Trace.WriteLine("Done."); } catch (Exception ex) { Trace.WriteLine(ex); } try { var multiplyBlock = new TransformBlock<int, int>(item => { // if (item == 1) // throw new InvalidOperationException("Blech."); return item * 2; }); var subtractBlock = new TransformBlock<int, int>(item => item - 2); multiplyBlock.LinkTo(subtractBlock); var printing = new ActionBlock<int>(i => Console.WriteLine("Received {0}", i.ToString())); subtractBlock.LinkTo(printing, new DataflowLinkOptions { PropagateCompletion = true }); IObservable<int> obs = subtractBlock.AsObservable(); obs.Subscribe(i => Console.WriteLine("Received Observable {0}", i.ToString())); for(int i = 0; i < 10;i++){ multiplyBlock.Post(i); subtractBlock.Post(i); } Console.ReadLine (); //printing.Completion.Wait(); } catch (AggregateException exception) { AggregateException ex = exception.Flatten(); System.Diagnostics.Trace.WriteLine(ex.InnerException); } Console.WriteLine ("Hello World!"); Console.ReadLine (); }