public void LoadTesting_MultipleProcessors()
{
GlobalConfiguration.Setup(s => s.UseOptions(new Options
{
MaxItemsInQueue = 10,
MinProcessors = 10,
MaxProcessors = 50
}));
var pipelineId = Guid.NewGuid().ToString();
var config = new PipelineConfiguration
{
Id = pipelineId,
InputHandler = new HandlerNode("CustomInput"),
OutputHandlers = new List <HandlerNode>
{
new HandlerNode("ConsoleOutput")
{
Filters = new List <string>
{
"Message -> msg",
"Level -> lvl"
}
},
new HandlerNode(typeof(ThreadWaitHandler))
}
};
var builder = new PipelineBuilder();
builder.BuildPipeline(config);
var result = ProfilerSession.StartSession()
.Task(ctx =>
{
var client = new EventDispatcher();
client.Process(pipelineId, new LogMessage {
Level = "Info", Message = $"Loadtesting {ctx.Get<int>(ContextKeys.Iteration)} on Thread {ctx.Get<int>(ContextKeys.ThreadId)}", Title = "Loadtest"
});
})
.SetIterations(100)
.SetThreads(10)
.Settings(s => s.RunWarmup = false)
.RunSession();
ProcessingServer.Server.WaitAll(pipelineId);
// delay to ensure all threads are ended
System.Threading.Tasks.Task.Delay(3000);
var storage = GlobalConfiguration.Configuration.Resolve <IStorage>();
Assert.GreaterOrEqual(100 * 10, storage.Get <long>(new StorageKey(pipelineId, "ProcessedEventsMetric")));
var stats = new StatisticsApi(pipelineId);
Assert.AreEqual(10, stats.GetMetricValue(MetricType.ThreadCount));
result.Trace("### LoadTesting");
}
public void EventBus_DefaultThread()
{
var eventBus = new EventBus(() => null, "EventBus_4");
var stats = new StatisticsApi("EventBus_4");
Assert.AreEqual(stats.GetMetricValue(MetricType.ThreadCount), 1);
}
public void EventBus_Close()
{
var eventBus = new EventBus(() => null, "EventBus_5");
eventBus.Close();
var stats = new StatisticsApi("EventBus_5");
Assert.IsTrue((int)stats.GetMetricValue(MetricType.QueueSize) == 0);
}
public void StatisticsAPi_GetMetricValue_ProcessedEvents()
{
var storage = new Mock <IStorage>();
storage.Setup(exp => exp.Get <Metric>(It.Is <StorageKey>(k => k.Key == $"metric:{MetricType.ProcessedEvents}"))).Returns(() => new Metric(MetricType.ProcessedEvents, "stats", 5));
storage.Setup(exp => exp.GetKeys(It.Is <StorageKey>(k => k.Key == $"metric:"))).Returns(() => new[] { $"metric:{MetricType.ProcessedEvents}" });
var statistics = new StatisticsApi("stats_checkmetricvalue", storage.Object);
Assert.AreEqual(5, statistics.GetMetricValue(MetricType.ProcessedEvents));
}
public void StatisticsAPi_GetMetricValue_ThreadCount()
{
var storage = new Mock <IStorage>();
storage.Setup(exp => exp.Get <Metric>(It.Is <StorageKey>(k => k.Key == $"metric:{MetricType.ThreadCount}"))).Returns(() => new Metric(MetricType.ThreadCount, "stats", true));
storage.Setup(exp => exp.GetKeys(It.Is <StorageKey>(k => k.Key == $"metric:"))).Returns(() => new[] { $"metric:{MetricType.ThreadCount}" });
var statistics = new StatisticsApi("stats_checkmetricvalue", storage.Object);
Assert.AreEqual(true, statistics.GetMetricValue(MetricType.ThreadCount));
}
public void StatisticsApi_Registered_EventLog()
{
var storage = new Mock <IStorage>();
storage.Setup(exp => exp.GetList <LogEvent>(It.Is <StorageKey>(k => k.Key == $"logs"))).Returns(() => new[] { new LogEvent() });
var stats = new StatisticsApi("EventBus_2", storage.Object);
Assert.IsNotNull(stats.GetMetricValue(MetricType.EventLog));
}
public void StatisticsApi_Registered_WorkersLog()
{
var storage = new Mock <IStorage>();
storage.Setup(exp => exp.GetList <ActiveWorkersLogMessage>(It.Is <StorageKey>(k => k.Key == $"logs:{MetricType.WorkersLog}"))).Returns(() => new[] { new ActiveWorkersLogMessage() });
var stats = new StatisticsApi("EventBus_2", storage.Object);
Assert.IsNotNull(stats.GetMetricValue(MetricType.WorkersLog));
}
public StatisticsClientApi(String apiKey, String baseUrl)
{
this.baseUrl = baseUrl;
api = new StatisticsApi(baseUrl + "/statistics/v3");
api.Configuration.AddDefaultHeader("x-api-key", apiKey);
api.Configuration.ApiClient.RestClient.CookieContainer = CookieManager.Cookies;
notifications = new Notifications();
}
/// <summary>
/// Get the value of a MetricType
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="metrics"></param>
/// <param name="type"></param>
/// <returns></returns>
public static T GetMetricValue <T>(this StatisticsApi metrics, MetricType type)
{
var metric = metrics.GetMetricValue(type);
if (metric == null)
{
return(default(T));
}
return((T)metric);
}
public void LoadTest_HeavyWork()
{
// start with the warmup...
var cnt = 0;
var proc = 0;
var pipelineId = Guid.NewGuid().ToString();
var server = new ProcessingServer();
server.Register(pipelineId, () =>
{
var pipeline = new EventPipeline();
pipeline.AddHandler(new LoadTestOuptuHandler(e =>
{
proc += 1;
// do some heavy work
Thread.Sleep(100);
}));
return(pipeline);
});
server.Register(pipelineId, new InputHandler <InputItem>());
var client = new EventDispatcher(server);
var profiler = ProfilerSession.StartSession()
.SetIterations(1000)
.SetThreads(5)
.Task(() =>
{
lock (pipelineId)
{
cnt += 1;
}
client.Process(pipelineId, new InputItem
{
Value = "StaticServer",
Name = "test",
Number = cnt
});
}).RunSession();
server.WaitAll(pipelineId);
profiler.Trace();
var monitor = new StatisticsApi(pipelineId);
var processed = monitor.GetMetricValue <long>(MetricType.ProcessedEvents);
Assert.AreEqual(processed, cnt, "Processed events are not equal to sent events {0} to {1}", processed, cnt);
}
public void Arrange()
{
_fakeHandler = new FakeResponseHandler();
var httpClient = new HttpClient(_fakeHandler);
var config = new Mock <ICommitmentsApiClientConfiguration>();
config.Setup(m => m.BaseUrl).Returns(ExpectedApiBaseUrl);
_statisticsApiClient = new StatisticsApi(httpClient, config.Object);
}
public EncodingApi(IBitmovinApiClientFactory apiClientFactory)
{
Inputs = new InputsApi(apiClientFactory);
Outputs = new OutputsApi(apiClientFactory);
Configurations = new ConfigurationsApi(apiClientFactory);
Filters = new FiltersApi(apiClientFactory);
Encodings = new EncodingsApi(apiClientFactory);
Manifests = new ManifestsApi(apiClientFactory);
Infrastructure = new InfrastructureApi(apiClientFactory);
Statistics = new StatisticsApi(apiClientFactory);
ErrorDefinitions = new ErrorDefinitionsApi(apiClientFactory);
}
public void EventBus_Process_WithouPipeline()
{
var eventBus = new EventBus(() => null, "EventBus_3");
eventBus.Publish(new Event("EventBus_3", "data"));
eventBus.WaitAll();
// event was not processed
var stats = new StatisticsApi("EventBus_3");
Assert.IsTrue((int)stats.GetMetricValue(MetricType.QueueSize) == 1);
}
public void Initialize(StatisticsApi api)
{
WebHeaderCollection headers = new WebHeaderCollection();
foreach (string key in api.Configuration.DefaultHeader.Keys)
{
switch (key)
{
case "x-api-key":
case "Authorization":
headers.Add(key, api.Configuration.DefaultHeader[key]);
break;
}
}
CookieCollection cookieCollection = CookieManager.Cookies.GetCookies(new Uri(api.GetBasePath()));
/**
* GWS currently only supports LongPolling as a method to receive events.
* So tell the CometD library to negotiate a handshake with GWS and setup a LongPolling session.
*/
LongPollingTransport transport = new LongPollingTransport(null);
transport.CookieCollection = cookieCollection;
transport.HeaderCollection = headers;
bayeuxClient = new BayeuxClient(api.GetBasePath() + "/notifications", new List <CometD.NetCore.Client.Transport.ClientTransport>()
{
transport
});
bayeuxClient.Handshake();
bayeuxClient.WaitFor(30000, new List <BayeuxClient.State>()
{
BayeuxClient.State.Connected
});
if (bayeuxClient.Connected)
{
foreach (Cookie cookie in cookieCollection)
{
CookieManager.AddCookie(cookie);
}
foreach (string channelName in subscriptions.Keys)
{
IClientSessionChannel channel = bayeuxClient.GetChannel(channelName);
channel.Subscribe(this);
}
}
}
public void EventBus_StatisticsApi_DefautlMetrics()
{
GlobalConfiguration.Setup(c => c.Register <IStorage>(new InmemoryStorage()));
var eventBus = new EventBus(() => null, "EventBus_1");
var stats = new StatisticsApi("EventBus_1");
Assert.AreEqual(5, stats.Count());
Assert.AreEqual(1, stats.GetMetricValue(MetricType.ThreadCount));
Assert.AreEqual(0, stats.GetMetricValue(MetricType.QueueSize));
Assert.AreEqual(0, stats.GetMetricValue(MetricType.ProcessedEvents));
Assert.IsNotNull(stats.GetMetricValue(MetricType.EventLog));
Assert.IsNotNull(stats.GetMetricValue(MetricType.WorkersLog));
GlobalConfiguration.Setup(c => c.Register <IStorage>(new InmemoryStorage()));
}
public void EventBus_StatisticsApi_Registered_Metrics()
{
GlobalConfiguration.Setup(c => c.Register <IStorage>(new InmemoryStorage()));
var stats = new StatisticsApi("EventBus_2");
Assert.IsNull(stats.GetMetricValue(MetricType.ThreadCount));
Assert.IsNull(stats.GetMetricValue(MetricType.QueueSize));
Assert.IsNull(stats.GetMetricValue(MetricType.ProcessedEvents));
var eventBus = new EventBus(() => null, "EventBus_2");
eventBus.Publish(new Event("EventBus_2", new EventData()));
stats = new StatisticsApi("EventBus_2");
Assert.IsNotNull(stats.GetMetricValue(MetricType.ThreadCount));
Assert.IsNotNull(stats.GetMetricValue(MetricType.QueueSize));
Assert.IsNotNull(stats.GetMetricValue(MetricType.ProcessedEvents));
GlobalConfiguration.Setup(c => c.Register <IStorage>(new InmemoryStorage()));
}
private async Task <ResponsePage <Metric> > ListMetricsFunc(MetricQueryOptions options)
{
try
{
var resp = await StatisticsApi.V3MetricsGetAsync(
include : options.Include,
interval : options.Interval,
start : options.Start,
end : options.End,
period : options.Period,
limit : options.Limit,
after : options.After,
order : options.Order);
var responsePage = new ResponsePage <Metric>(after: resp.After, hasMore: resp.HasMore, totalCount: resp.TotalCount);
responsePage.MapData <statistics.Model.Metric>(resp.Data, Metric.Map);
return(responsePage);
}
catch (statistics.Client.ApiException e)
{
throw new CloudApiException(e.ErrorCode, e.Message, e.ErrorContent);
}
}
public void StatisticsAPi_Ctor()
{
var statistics = new StatisticsApi("stats_ctor");
}
private PipelineMetric Monitor(PipelineModel pipeline, IStorage storage)
{
var defaultMetrics = new List <MetricType> {
MetricType.ThreadCount, MetricType.QueueSize, MetricType.ProcessedEvents
};
var server = pipeline.ServerName ?? Environment.MachineName;
var heartbeat = storage.Get <ServerModel>(new StorageKey($"server:{server.ToLower()}"));
var statistics = new StatisticsApi(server, pipeline.PipelineId, storage);
var metrics = new PipelineMetric(pipeline.PipelineId, server, heartbeat != null ? DateTime.Parse(heartbeat.Heartbeat) : (DateTime?)null);
metrics.AddMetric("Server", "Server", server);
metrics.AddMetric("Heartbeat", "Heartbeat", heartbeat != null ? DateTime.Parse(heartbeat.Heartbeat).ToString("yyyy.MM.dd HH:mm:ss") : "");
foreach (var key in defaultMetrics)
{
var metric = statistics.FirstOrDefault(s => s.Key == key);
if (metric == null)
{
metric = new Metric(key, "", "");
}
metrics.AddMetric(key.ToString(), metric?.Title, metric?.Value);
}
foreach (var metric in statistics)
{
if (defaultMetrics.Contains(metric.Key))
{
continue;
}
switch (metric.Key)
{
case MetricType.EventLog:
//LogEventStatisticWriter.cs
if (metric.Value is IEnumerable <LogEvent> logs)
{
foreach (var log in logs.OrderByDescending(l => l.Timestamp).Take(Options.LogCount))
{
metrics.AddElement(metric.Key.ToString(), metric.Title, new DashboardLog
{
Timestamp = log.Timestamp,
Source = log.Source,
Level = log.Level.ToString(),
Message = log.Message
});
}
}
break;
case MetricType.WorkersLog:
//WorkersLogMetricCounter.cs
if (metric.Value is IEnumerable <ActiveWorkersLogMessage> workers)
{
foreach (var worker in workers.OrderByDescending(w => w.Timestamp).Take(Options.WorkersLogCount))
{
metrics.AddElement(metric.Key.ToString(), metric.Title, new TimelineLog <int>
{
Timestamp = worker.Timestamp,
Value = worker.ActiveWorkers
});
}
}
break;
default:
metrics.AddMetric(metric.Key.ToString(), metric.Title, metric.Value ?? 0);
break;
}
}
return(metrics);
}
public StatisticsApiTests()
{
instance = new StatisticsApi();
}
public void Init()
{
instance = new StatisticsApi();
}
