static void Main(string[] args)
{
//使用Rx Observable对象封装async代码
//将Task<T>转换为IObservable<T>
var client = new HttpClient();
IObservable <HttpResponseMessage> response = client.GetAsync("http://kworb.net/pop/").ToObservable();
//另一种方法是调用StartAsync
IObservable <HttpResponseMessage> response2 = Observable.StartAsync(token => client.GetAsync("http://kworb.net/pop/", token));
//这两个方法都会立即启动异步操作，如果想要其接受订阅后启动，可使用FromAsync
IObservable <HttpResponseMessage> response3 = Observable.FromAsync(token => client.GetAsync("http://kworb.net/pop/", token));
//Rx Observable对象和数据流网络
var buffer = new BufferBlock <int>();
IObservable <int> integers = buffer.AsObservable(); //创建一个缓冲块到Observable对象的接口
integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("done"));
//如果使用一个网格并把它作为观察流的目的，情况会稍微复杂一点。
//ticks->display->done
IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)).Timestamp().Select(x => x.Timestamp).Take(5);
var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x));
ticks.Subscribe(display.AsObserver());
try
{
display.Completion.Wait();
Console.WriteLine("Done.");
}catch (Exception ex)
{
Console.WriteLine(ex);
}
}
private static void TdfRxGroupBy()
{
var buffer = new BufferBlock <int>();
Task.Run(() =>
{
for (int i = 0; i < 25; i++)
{
buffer.Post(i);
Thread.Sleep(500);
}
});
Console.WriteLine("Start");
Thread.Sleep(3000);
var groups = from item in buffer.AsObservable()
group item by item % 5 into g
select g;
groups.Subscribe(g =>
{
ConsoleColor c = (ConsoleColor)(g.Key + 10);
var ab = new ActionBlock <int>(item =>
{
lock (groups)
{
Console.ForegroundColor = c;
Console.Write("{0},", item);
}
});
g.Subscribe(ab.AsObserver());
});
}
static void Main(string[] args)
{
// 考虑把数据流块用作可观察流的输入
var buffer = new BufferBlock <int>();
IObservable <int> integers = buffer.AsObservable();
integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("done"));
buffer.Post(13);
buffer.Complete();
buffer.Completion.Wait();
// 考虑使用数据流网络并把它当作可观察流的终点
IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1))
.Timestamp()
.Select(x => x.Timestamp)
.Take(5);
var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x));
ticks.Subscribe(display.AsObserver());
try
{
display.Completion.Wait();
Console.WriteLine("Done");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public static Task ExecuteInGroupsAsync <T>(
this IEnumerable <T> source, Func <T, Task> func, int groupSize)
{
var block = new ActionBlock <IEnumerable <T> >(
g => Task.WhenAll(g.Select(func)));
source.ToObservable()
.Buffer(groupSize)
.Subscribe(block.AsObserver());
return(block.Completion);
}
public ElasticSearchOutput(ElasticSearchOutputConfig config, PutIndexTemplateDescriptor indexTemplate = null)
{
_logger = LogProvider.For <ElasticSearchOutput>();
_logger.Info("Creating");
var connectionSettings = new ConnectionSettings(new Uri(config.ElasticSearchUri))
.RequestTimeout(config.RetryInterval)
.MaximumRetries(config.RetryCount);
_elasticClient = new ElasticClient(connectionSettings);
var batchCount = config.MaxBatchCount > 0 ? config.MaxBatchCount : 1;
_uploadBlock = new ActionBlock <IList <TelemetryEvent> >(list => Send(list, throwExceptions: false), new ExecutionDataflowBlockOptions
{
BoundedCapacity = config.BoundedCapacity,
MaxDegreeOfParallelism = config.MaxDegreeOfParallelism,
});
_internalStreamBufferSubscription = _internalStream
.Buffer(TimeSpan.FromSeconds(config.BufferTimeSec), batchCount)
.Subscribe(_uploadBlock.AsObserver());
_jsonSerializerSettings = new JsonSerializerSettings()
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
_elasticClient.PutIndexTemplate(indexTemplate ?? new PutIndexTemplateDescriptor(config.ElasticSearchIndexName + "-template")
.Template(config.ElasticSearchIndexName + "*")
.Settings(x => x
.NumberOfReplicas(0)
.NumberOfShards(1))
.Mappings(m => m
.Map("_default_", tm => tm
.AllField(af => af.Enabled(false))
.DynamicTemplates(d => d
.DynamicTemplate("all_strings_not_analyzed", dd => dd
.MatchMappingType("string")
.Match("*")
.Mapping(dm => dm
.String(sm => sm
.NotAnalyzed()))))
.Properties(p => p
.String(sp => sp
.Name("message")
.Index(FieldIndexOption.Analyzed))))));
_config = config;
}
public void ExampleObservableToActionBlock()
{
IObservable <long> ticks =
Observable.Interval(TimeSpan.FromMilliseconds(1))
.Select(t => t)
.Take(5);
var block = new ActionBlock <long>(x => Console.WriteLine($"ActionBlock: {x}"));
ticks.Subscribe(block.AsObserver());
block.Completion.Wait();
}
public ElasticSearchOutput(ElasticSearchOutputConfig config, PutIndexTemplateDescriptor indexTemplate = null)
{
_logger = LogProvider.For<ElasticSearchOutput>();
_logger.Info("Creating");
var connectionSettings = new ConnectionSettings(new Uri(config.ElasticSearchUri))
.RequestTimeout(config.RetryInterval)
.MaximumRetries(config.RetryCount);
_elasticClient = new ElasticClient(connectionSettings);
var batchCount = config.MaxBatchCount > 0 ? config.MaxBatchCount : 1;
_uploadBlock = new ActionBlock<IList<TelemetryEvent>>(list => Send(list, throwExceptions: false), new ExecutionDataflowBlockOptions
{
BoundedCapacity = config.BoundedCapacity,
MaxDegreeOfParallelism = config.MaxDegreeOfParallelism,
});
_internalStreamBufferSubscription = _internalStream
.Buffer(TimeSpan.FromSeconds(config.BufferTimeSec), batchCount)
.Subscribe(_uploadBlock.AsObserver());
_jsonSerializerSettings = new JsonSerializerSettings() { ContractResolver = new CamelCasePropertyNamesContractResolver() };
_elasticClient.PutIndexTemplate(indexTemplate ?? new PutIndexTemplateDescriptor(config.ElasticSearchIndexName+"-template")
.Template(config.ElasticSearchIndexName+"*")
.Settings(x => x
.NumberOfReplicas(0)
.NumberOfShards(1))
.Mappings(m => m
.Map("_default_", tm => tm
.AllField(af => af.Enabled(false))
.DynamicTemplates(d => d
.DynamicTemplate("all_strings_not_analyzed", dd => dd
.MatchMappingType("string")
.Match("*")
.Mapping(dm => dm
.String(sm => sm
.NotAnalyzed()))))
.Properties(p => p
.String(sp => sp
.Name("message")
.Index(FieldIndexOption.Analyzed))))));
_config = config;
}
//调用AsObserver让一个块订阅一个可观察流。
public static void ObservableToBlock()
{
IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)).Timestamp().Select(x => x.Timestamp).Take(5);
var display = new ActionBlock <DateTimeOffset>(x => Trace.WriteLine(x));
ticks.Subscribe(display.AsObserver());
try
{
display.Completion.Wait();
Trace.WriteLine("Dowe.");
}
catch (Exception ex)
{
Trace.WriteLine(ex);
}
}
public static void UseToObservable()
{
var ob = T().ToObservable();
var ob2 = Observable.FromAsync(T);
var buffer = new BufferBlock <int>();
var integers = buffer.AsObservable();
integers.Subscribe(data => Console.WriteLine(data),
ex => Console.WriteLine(ex),
() => Console.WriteLine("Done"));
buffer.Post(12);
var client = new HttpClient();
var response = client.GetAsync("http://www.baidu.com").ToObservable();
var response2 = Observable.StartAsync(token => client.GetAsync("http://www.baidu.com", token));
IObservable <string> urls = new List <string>()
{
"http://www.baidu.coM"
}.ToObservable();
var response3 = urls.SelectMany((url, token) => client.GetAsync(url, token));
var ticks = Observable.Interval(TimeSpan.FromSeconds(1))
.Timestamp()
.Select(x => x.Timestamp)
.Take(5);
var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x));
ticks.Subscribe(display.AsObserver());
try
{
display.Completion.Wait();
Console.WriteLine("Done");
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public async Task TestAsObservableAndAsObserver_DataPropagation()
{
// Test that preset data flows correctly
{
var bb = new BufferBlock<int>();
bb.PostRange(0, 2);
bb.Complete();
int nextValueExpected = 0;
var ab = new ActionBlock<int>(i => {
Assert.True(i == nextValueExpected, string.Format("Expected next value to be {0} but got {1}", nextValueExpected, i));
nextValueExpected++;
});
bb.AsObservable().Subscribe(ab.AsObserver());
await ab.Completion;
}
// Test that new data flows correctly
{
int nextValueExpected = -2;
var ab = new ActionBlock<int>(i => {
Assert.True(i == nextValueExpected, string.Format("Expected next value to be {0} but got {1}", nextValueExpected, i));
nextValueExpected++;
});
var bb = new BufferBlock<int>();
bb.AsObservable().Subscribe(ab.AsObserver());
bb.PostRange(-2, 0);
bb.Complete();
await ab.Completion;
}
// Test that unsubscribing stops flow of data and stops completion
{
var target = new BufferBlock<int>();
var source = new BufferBlock<int>();
using (source.AsObservable().Subscribe(target.AsObserver()))
{
source.PostItems(1, 2);
Assert.Equal(expected: 1, actual: await target.ReceiveAsync());
Assert.Equal(expected: 2, actual: await target.ReceiveAsync());
}
source.Post(3);
var wb = new WriteOnceBlock<int>(i => i);
source.LinkTo(wb);
await wb.Completion;
source.Complete();
await source.Completion;
Assert.False(target.Completion.IsCompleted);
}
}
public static void Main(string[] args)
{
var testObs = new TransformBlock <int, int>(item => item * item);
var o = testObs.AsObservable();
o.Subscribe(i => Console.WriteLine("Test Obs received {0}", i.ToString()));
for (int i = 0; i < 5; i++)
{
testObs.Post(i);
}
testObs.Completion.Wait();
Console.ReadKey();
var buffer = new BufferBlock <int>();
IObservable <int> integers = buffer.AsObservable();
integers.Subscribe(data =>
Console.WriteLine(data),
ex => Console.WriteLine(ex),
() => Console.WriteLine("Done"));
buffer.Post(13);
buffer.Post(14);
buffer.Post(15);
buffer.Post(16);
Console.ReadKey();
IObservable <DateTimeOffset> ticks =
Observable.Interval(TimeSpan.FromSeconds(1))
.Timestamp()
.Select(x => x.Timestamp)
.Take(5);
var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x));
ticks.Subscribe(display.AsObserver());
Console.ReadKey();
try
{
display.Completion.Wait();
Trace.WriteLine("Done.");
}
catch (Exception ex)
{
Trace.WriteLine(ex);
}
try
{
var multiplyBlock = new TransformBlock <int, int>(item =>
{
// if (item == 1)
// throw new InvalidOperationException("Blech.");
return(item * 2);
});
var subtractBlock = new TransformBlock <int, int>(item => item - 2);
multiplyBlock.LinkTo(subtractBlock);
var printing = new ActionBlock <int>(i => Console.WriteLine("Received {0}", i.ToString()));
subtractBlock.LinkTo(printing, new DataflowLinkOptions {
PropagateCompletion = true
});
IObservable <int> obs = subtractBlock.AsObservable();
obs.Subscribe(i => Console.WriteLine("Received Observable {0}", i.ToString()));
for (int i = 0; i < 10; i++)
{
multiplyBlock.Post(i);
subtractBlock.Post(i);
}
Console.ReadLine();
//printing.Completion.Wait();
}
catch (AggregateException exception)
{
AggregateException ex = exception.Flatten();
System.Diagnostics.Trace.WriteLine(ex.InnerException);
}
Console.WriteLine("Hello World!");
Console.ReadLine();
}
public static async Task Download(CloudStorageAccount storageAccount, DateTime startTimeInclusive, DateTime endTimeExclusive, StreamWriter writer, string cacheDirectory = null)
{
var telemetry = new TelemetryClient();
var stopwatch = Stopwatch.StartNew();
var blobClient = storageAccount.CreateCloudBlobClient();
var blobsToDownload = new List <DownloadItem>();
telemetry.TrackEvent($"Downloading {startTimeInclusive:yyyy-MM-dd} to {endTimeExclusive:yyyy-MM-dd}");
long downloadedBytes = 0;
var options = new BlobRequestOptions
{
MaximumExecutionTime = new TimeSpan(0, 60, 0),
ServerTimeout = new TimeSpan(0, 60, 0),
RetryPolicy = new ExponentialRetry()
};
var downloadBlock = new ActionBlock <DownloadItem>(async item =>
{
try
{
if (cacheDirectory != null)
{
Console.WriteLine($"Downloading {item.Blob.Uri}");
}
var temp = item.Filename + ".temp";
await item.Blob.DownloadToFileAsync(temp, FileMode.Create, AccessCondition.GenerateEmptyCondition(), options, new OperationContext())
.ContinueWith(t => File.Move(temp, item.Filename));
Interlocked.Add(ref downloadedBytes, item.Blob.Properties.Length);
}
catch (Exception e)
{
throw new Exception("Failed to download: " + item.Blob.Uri, e);
}
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = 16, MaxDegreeOfParallelism = 8
});
var downloadInput = downloadBlock.AsObserver();
// enumerate full days
for (var currentDateTime = new DateTime(startTimeInclusive.Year, startTimeInclusive.Month, startTimeInclusive.Day);
currentDateTime < endTimeExclusive;
currentDateTime += TimeSpan.FromDays(1))
{
foreach (var blob in blobClient.ListBlobs($"joined-examples/{currentDateTime:yyyy/MM/dd}", useFlatBlobListing: true).OfType <CloudBlockBlob>())
{
// find relevant folder
var match = Regex.Match(blob.Uri.AbsolutePath, @"/(?<hour>\d{2})/(?<name>[^/]+\.json)$");
if (!match.Success)
{
telemetry.TrackTrace($"Skipping nvalid blob '{blob.Uri}'.");
continue;
}
var blobDateTime = currentDateTime + TimeSpan.FromHours(int.Parse(match.Groups["hour"].Value, CultureInfo.InvariantCulture));
if (!(blobDateTime >= startTimeInclusive && blobDateTime < endTimeExclusive))
{
continue;
}
var item = new DownloadItem {
Blob = blob, DateTime = blobDateTime
};
blobsToDownload.Add(item);
if (cacheDirectory != null)
{
Console.WriteLine($"Checking {currentDateTime:yyyy/MM/dd}{match.Value}...");
// create directory
var outputDir = Path.Combine(cacheDirectory, $"{blobDateTime:yyyy/MM/dd/HH}");
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
var outputFile = Path.Combine(outputDir, match.Groups["name"].Value);
item.Filename = outputFile;
if (!File.Exists(item.Filename))
{
downloadInput.OnNext(item);
}
}
}
}
downloadInput.OnCompleted();
await downloadBlock.Completion;
if (cacheDirectory != null)
{
Console.WriteLine($"Download time: {stopwatch.Elapsed}");
}
foreach (var blobGroup in blobsToDownload.GroupBy(i => i.DateTime).OrderBy(i => i.Key))
{
if (cacheDirectory != null)
{
Console.WriteLine($"Merging {blobGroup.Key:yyyy/MM/dd HH}");
}
// TODO: only use blob
var dsBlobs = blobGroup.Select(b => new DecisionServiceBlob(b)).ToArray();
// not really faster and more error prone
//var priorityQueue = new FastPriorityQueue<DecisionServiceEvent>(dsBlobs.Length);
var priorityQueue = new SimplePriorityQueue <DecisionServiceEvent>();
foreach (var di in dsBlobs)
{
var evt = await di.Receive();
if (evt != null)
{
evt.Blob = di;
priorityQueue.Enqueue(evt, evt.DateTime.Ticks);
}
}
while (true)
{
var top = priorityQueue.First;
// add VW string format
writer.WriteLine(top.Line);
if (await top.Next())
{
priorityQueue.UpdatePriority(top, top.DateTime.Ticks);
}
else
{
priorityQueue.Dequeue();
if (priorityQueue.Count == 0)
{
break;
}
}
}
}
if (cacheDirectory != null)
{
Console.WriteLine($"Total time: {stopwatch.Elapsed}");
}
}
public static void Main(string[] args)
{
var testObs = new TransformBlock<int, int>(item => item * item);
var o = testObs.AsObservable ();
o.Subscribe(i=> Console.WriteLine("Test Obs received {0}", i.ToString()));
for (int i = 0; i < 5; i++) {
testObs.Post (i);
}
testObs.Completion.Wait ();
Console.ReadKey ();
var buffer = new BufferBlock<int>();
IObservable<int> integers = buffer.AsObservable();
integers.Subscribe(data =>
Console.WriteLine(data),
ex => Console.WriteLine(ex),
() => Console.WriteLine("Done"));
buffer.Post(13);
buffer.Post(14);
buffer.Post(15);
buffer.Post(16);
Console.ReadKey ();
IObservable<DateTimeOffset> ticks =
Observable.Interval(TimeSpan.FromSeconds(1))
.Timestamp()
.Select(x => x.Timestamp)
.Take(5);
var display = new ActionBlock<DateTimeOffset>(x => Console.WriteLine(x));
ticks.Subscribe(display.AsObserver());
Console.ReadKey ();
try
{
display.Completion.Wait();
Trace.WriteLine("Done.");
}
catch (Exception ex)
{
Trace.WriteLine(ex);
}
try
{
var multiplyBlock = new TransformBlock<int, int>(item =>
{
// if (item == 1)
// throw new InvalidOperationException("Blech.");
return item * 2;
});
var subtractBlock = new TransformBlock<int, int>(item => item - 2);
multiplyBlock.LinkTo(subtractBlock);
var printing = new ActionBlock<int>(i => Console.WriteLine("Received {0}", i.ToString()));
subtractBlock.LinkTo(printing, new DataflowLinkOptions { PropagateCompletion = true });
IObservable<int> obs = subtractBlock.AsObservable();
obs.Subscribe(i => Console.WriteLine("Received Observable {0}", i.ToString()));
for(int i = 0; i < 10;i++){
multiplyBlock.Post(i);
subtractBlock.Post(i);
}
Console.ReadLine ();
//printing.Completion.Wait();
}
catch (AggregateException exception)
{
AggregateException ex = exception.Flatten();
System.Diagnostics.Trace.WriteLine(ex.InnerException);
}
Console.WriteLine ("Hello World!");
Console.ReadLine ();
}
