public void TestDefaults()
{
var q = new SampleStream(Invariants.Targeted(new[] { new QuantileEpsilonPair(0.99d, 0.001d) }));
double g = q.Query(0.99);
Assert.Equal(double.NaN, g);
}
private void VerifyPercsWithAbsoluteEpsilon(double[] a, SampleStream s, IReadOnlyList <QuantileEpsilonPair> targets)
{
foreach (var target in targets)
{
double n = a.Length;
int k = (int)(target.Quantile * n);
if (k < 1)
{
k = 1;
}
int lower = (int)((target.Quantile - target.Epsilon) * n);
if (lower < 1)
{
lower = 1;
}
int upper = (int)Math.Ceiling((target.Quantile + target.Epsilon) * n);
if (upper > a.Length)
{
upper = a.Length;
}
double w = a[k - 1];
double min = a[lower - 1];
double max = a[upper - 1];
double g = s.Query(target.Quantile);
Assert.True(g >= min, $"q={target.Quantile}: want {w} [{min}, {max}], got {g}");
Assert.True(g <= max, $"q={target.Quantile}: want {w} [{min}, {max}], got {g}");
}
}
public SampleStreamProcessor(IConfiguration configuration,
ITweetRepository tweetRepository,
ICacheRepository cacheRepository,
ILogger <SampleStreamProcessor> logger,
IEmojiProcessor emojiProcessor,
IHashtagProcessor hashtagProcessor)
{
Logger = logger;
EmojiProcessor = emojiProcessor;
HashtagProcessor = hashtagProcessor;
TweetRepository = tweetRepository;
CacheRepository = cacheRepository;
Configuration = configuration;
TwitterCredentialsOptions = configuration.GetSection(TwitterCredentialsOptions.TwitterCredentials).Get <TwitterCredentialsOptions>();
SampleStreamOptions = configuration.GetSection(SampleStreamOptions.SampleStream).Get <SampleStreamOptions>();
UserClient = new TwitterClient(TwitterCredentialsOptions.ConsumerKey, TwitterCredentialsOptions.ConsumerSecret, TwitterCredentialsOptions.AccessToken, TwitterCredentialsOptions.AccessSecret);
SampleStream = UserClient.Streams.CreateSampleStream();
SampleStream.AddLanguageFilter(SampleStreamOptions.LanguageFilter);
SampleStream.FilterLevel = (StreamFilterLevel)SampleStreamOptions.FilterLevel;
SampleStream.TweetReceived += (sender, eventArgs) =>
{
Tweets.Add((Tweet)eventArgs.Tweet);
if (NumberOfTotalTweetsPerSession > SampleStreamOptions.MaxAllowedTweetsLimit)
{
StopStream();
}
if (Tweets.Count > SampleStreamOptions.TweetsPageSize || GetStreamStateAsync().Result == StreamStateEnum.Stop)
{
ProcessTweetsAsync();
}
};
}
private void StartStream <THub>(SampleStream stream, IHubContext <THub> hubContext) where THub : Hub
{
stream.Queue.ContentAdded += (a, b) =>
{
hubContext.Clients.All.InvokeAsync("send", b);
};
stream.StartAsync();
}
public void TestLowBiasedQuery()
{
var random = new Random(42);
var s = new SampleStream(Invariants.LowBiased(_relativeEpsilon));
var a = PopulateStream(s, random);
VerifyLowPercsWithRelativeEpsilon(a, s);
}
public void TestUncompressedOne()
{
var q = new SampleStream(Invariants.Targeted(new[] { new QuantileEpsilonPair(0.99d, 0.001d) }));
q.InsertRange(new[] { 3.14 });
double g = q.Query(0.90);
Assert.Equal(3.14, g);
}
public void TestTargetedQuery(double quantile, double epsilon, double[] data)
{
Array.Sort(data);
var targets = new[] { new QuantileEpsilonPair(quantile, epsilon) };
var s = new SampleStream(Invariants.Targeted(targets));
s.InsertRange(data);
VerifyPercsWithAbsoluteEpsilon(data, s, targets);
}
private SampleStream CreateTrackedStream(string track)
{
this.Running = true;
var oq = new OrderedQueue();
var ct = new SampleStream(credentials, oq)
.AddLanguage(Language.English)
.AddTrack(track)
.AddQueryParameter("result_type", "recent");
return(ct);
}
public void StopStream()
{
try
{
if (SampleStream.StreamState != StreamState.Stop)
{
SampleStream.Stop();
Logger.LogInformation("SampledStream Stopped");
}
}
catch (Exception ex)
{
Logger.LogError("Error occurred at StopStream", ex);
}
}
protected void HandleRequest()
{
Response.StatusCode = 200;
Response.Headers.Add("Connection", "keep-alive");
Response.Headers.Add("Content-Type", "text/event-stream;charset=UTF-8");
Response.Headers.Add("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate");
Response.Headers.Add("Pragma", "no-cache");
SampleSubscription = SampleStream
.ObserveOn(NewThreadScheduler.Default)
.Subscribe(
async(sampleDataAsString) =>
{
if (sampleDataAsString != null)
{
try
{
await Response.WriteAsync("data: " + sampleDataAsString + "\n\n").ConfigureAwait(false);
await Response.Body.FlushAsync().ConfigureAwait(false);
}
catch (Exception)
{
if (SampleSubscription != null)
{
SampleSubscription.Dispose();
SampleSubscription = null;
}
}
}
},
(error) =>
{
if (SampleSubscription != null)
{
SampleSubscription.Dispose();
SampleSubscription = null;
}
},
() =>
{
if (SampleSubscription != null)
{
SampleSubscription.Dispose();
SampleSubscription = null;
}
});
Response.Body.FlushAsync();
}
public void StartCapture()
{
//Checks to make sure that the camera has not already been started
if (Status != CameraStatus.Stopped)
{
throw new InvalidOperationException("A camera cannot be started if it is already running");
}
//Updates the status
Status = CameraStatus.Running;
//Initialize the sense manager and streams
SampleStream = SampleReader.Activate(SMInstance);
SampleStream.EnableStream(StreamType.STREAM_TYPE_COLOR, WIDTH, HEIGHT, FPS, StreamOption.STREAM_OPTION_STRONG_STREAM_SYNC);
SampleStream.EnableStream(StreamType.STREAM_TYPE_DEPTH, WIDTH, HEIGHT, FPS, StreamOption.STREAM_OPTION_STRONG_STREAM_SYNC);
SMInstance.Init();
}
public async Task StartStreamAsync()
{
try
{
if (this.SampleStream.StreamState != StreamState.Running)
{
StartTime = DateTime.Now;
NumberOfTotalTweetsPerSession = 0;
SampleStream.StartAsync();
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Error occurred during stream processing", null);
StopStream();
}
}
public void StopCapture()
{
//Checks to make sure that the camera has not already been stopped before starting it again
if (Status != CameraStatus.Running)
{
throw new InvalidOperationException("A camera cannot be stopped if it is not running");
}
//Updates the status
Status = CameraStatus.Stopped;
//Dispose of the created stream
SampleStream.Dispose();
SampleStream = null;
//Close the sense manager
SMInstance.Close();
}
private void VerifyHighPercsWithRelativeEpsilon(double[] a, SampleStream s)
{
Array.Sort(a);
foreach (double qu in _highQuantiles)
{
double n = a.Length;
int k = (int)(qu * n);
int lowerRank = (int)((1 - ((1 + _relativeEpsilon) * (1 - qu))) * n);
int upperRank = (int)Math.Ceiling((1 - ((1 - _relativeEpsilon) * (1 - qu))) * n);
double w = a[k - 1];
double min = a[lowerRank - 1];
double max = a[upperRank - 1];
double g = s.Query(qu);
Assert.True(g >= min, $"q={qu}: want {w} [{min}, {max}], got {g}");
Assert.True(g <= max, $"q={qu}: want {w} [{min}, {max}], got {g}");
}
}
private static double[] PopulateStream(SampleStream stream, Random random)
{
var a = new double[100100];
for (int i = 0; i < a.Length; i++)
{
double v = random.NormDouble();
// Add 5% asymmetric outliers.
if (i % 20 == 0)
{
v = (v * v) + 1;
}
a[i] = v;
}
Array.Sort(a);
stream.InsertRange(a);
return(a);
}
