public void given_various_sizes()
{
Assert.AreEqual("10 Bytes", FormatEvil.SizeInBytes(10));
Assert.AreEqual("10 KB", FormatEvil.SizeInBytes(10 * 1024));
Assert.AreEqual("72 MB", FormatEvil.SizeInBytes(72 * 1024 * 1024));
Assert.AreEqual("72 GB", FormatEvil.SizeInBytes(72L * 1024 * 1024 * 1024));
}
public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
var maxCount = 10000;
if (args.Length > 0)
{
int.TryParse(args[0], out maxCount);
}
var sw = Stopwatch.StartNew();
var records = context.Client.EventStores.ReadAllEvents(EventStoreOffset.Zero, maxCount);
int msgCount = 0;
long dataSize = 0;
foreach (var record in records)
{
msgCount += 1;
dataSize += record.EventData.Length;
}
sw.Stop();
var speed = dataSize / sw.Elapsed.TotalSeconds;
var messageSpeed = (int)(msgCount / sw.Elapsed.TotalSeconds);
context.Log.Info("{0} msgPerSec or {1}", messageSpeed, FormatEvil.SpeedInBytes(speed));
PerfUtils.LogTeamCityGraphData(string.Format("EN_{0}_msgPerSec", maxCount), messageSpeed);
PerfUtils.LogTeamCityGraphData(string.Format("EN_{0}_bytesPerSec", maxCount), (int)speed);
return(true);
}
HashSet <string> ImportBatch(CommandProcessorContext context, string streamId, int batchCount, int batchSize)
{
var result = new HashSet <string>();
int totalBytes = 0;
var watch = Stopwatch.StartNew();
for (int i = 0; i < batchCount; i++)
{
string message = string.Format(singleThreadMessageTemplate, i);
context.Client.EventStores.WriteEventsInLargeBatch("",
Enumerable.Range(0, batchSize).Select(
x =>
{
try
{
var bytes = Encoding.UTF8.GetBytes(streamId + string.Format(message, x));
totalBytes += bytes.Length;
return(bytes);
}
catch (Exception ex)
{
context.Log.Error(ex.Message);
throw ex;
}
}));
for (int j = 0; j < batchSize; j++)
{
result.Add(string.Format(message, j));
}
}
var totalMs = watch.ElapsedMilliseconds;
var byteSize = batchSize * batchCount > 0 ? totalBytes / (batchCount * batchSize) : 0;
var key = string.Format("WB-{0}-{1}-{2}-{3}-bytesPerSec", 1, batchCount, batchSize, byteSize);
var bytesPerSec = totalMs > 0 ? (totalBytes * 1000D / totalMs) : 0;
context.Log.Debug("Throughput: {0}", FormatEvil.SpeedInBytes(bytesPerSec));
PerfUtils.LogTeamCityGraphData(key, (int)bytesPerSec);
return(result);
}
public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
var batchCount = 15;
var batchSize = 10000;
var projectionCount = 10;
var readerCount = 10;
if (args.Length > 0)
{
int.TryParse(args[0], out batchCount);
}
if (args.Length > 1)
{
int.TryParse(args[1], out batchSize);
}
if (args.Length > 2)
{
int.TryParse(args[2], out projectionCount);
}
if (args.Length > 3)
{
int.TryParse(args[3], out readerCount);
}
context.Log.Debug("batchCount: {0}", batchCount);
context.Log.Debug("batchSize: {0}", batchSize);
context.Log.Debug("projectionCount: {0}", projectionCount);
context.Log.Debug("readerCount: {0}", readerCount);
var startEvt = new ManualResetEventSlim(false);
_failures = 0;
var streamId = "SmartAppTest-" + Guid.NewGuid();
var totalSw = Stopwatch.StartNew();
// write half of data to the stream
var writeStat = new Stat();
WriteEvents(streamId, batchCount / 2, batchSize, context, writeStat);
var wrBytesPerSec = writeStat.ElapsedMsec > 0 ? (writeStat.Count * 1000D / writeStat.ElapsedMsec) : 0;
context.Log.Debug("Events write throughput: {0}", FormatEvil.SpeedInBytes(wrBytesPerSec));
var threads = new List <Thread>();
using (var cts = new CancellationTokenSource())
using (var linked = CancellationTokenSource.CreateLinkedTokenSource(token, cts.Token))
{
// Upload some random data to an event stream in batches (a thread and large batches)
writeStat = new Stat();
var writerThread = CreateEventWriterThread(streamId, batchCount / 2, batchSize, context, writeStat, startEvt, linked.Token);
writerThread.Start();
// run a set of projections in parallel for this event stream (1 thread per projection)
var projStat = new Stat[projectionCount];
for (var i = 0; i < projectionCount; i++)
{
projStat[i] = new Stat();
threads.Add(CreateProjectionThread(streamId, i, context, startEvt, linked.Token, projStat[i]));
threads.Last().Start();
}
// poll projected views with multiple concurrent readers
var projectionIndex = 0;
var readerStat = new Stat[readerCount];
for (var i = 0; i < readerCount; i++)
{
readerStat[i] = new Stat();
threads.Add(CreateViewReaderThread(i, projectionIndex, context, startEvt, linked.Token, readerStat[i]));
threads.Last().Start();
projectionIndex++;
if (projectionIndex >= projectionCount)
{
projectionIndex = 0;
}
}
// Start all thread
startEvt.Set();
// Wait until second half of data will be written
writerThread.Join();
// Cancel rest threads
cts.Cancel();
foreach (var thread in threads)
{
thread.Join();
}
totalSw.Stop();
// Projections stat
wrBytesPerSec = writeStat.ElapsedMsec > 0 ? (writeStat.Count * 1000D / writeStat.ElapsedMsec) : 0;
context.Log.Debug("Events write throughput under load: {0}", FormatEvil.SpeedInBytes(wrBytesPerSec));
var elapsedMsec = projStat.Max(s => s.ElapsedMsec);
var totalBytes = projStat.Sum(s => s.Count);
var bytesPerSec = elapsedMsec > 0 ? (totalBytes * 1000D / elapsedMsec) : 0;
context.Log.Debug("Events read throughput: {0}", FormatEvil.SpeedInBytes(bytesPerSec));
elapsedMsec = readerStat.Max(s => s.ElapsedMsec);
var totalReads = readerStat.Sum(s => s.Count);
var readsPerSec = elapsedMsec > 0 ? (totalReads * 1000D / elapsedMsec) : 0;
context.Log.Debug("Views object read rate: {0} instance/sec", FormatEvil.ToHumanReadable(readsPerSec));
context.Log.Info("Total time: {0}", totalSw.Elapsed);
var key = string.Format("SAB-WR-{0}-{1}-bytesPerSec", batchCount, batchSize);
PerfUtils.LogTeamCityGraphData(key, (int)wrBytesPerSec);
PerfUtils.LogTeamCityGraphData("SAB-RE-bytesPerSec", (int)bytesPerSec);
PerfUtils.LogTeamCityGraphData("SAB-RV-objectsPerSec", (int)readsPerSec);
}
return(true);
}
public void Handle(ClientMessage.ImportEvents msg)
{
Log.Info("Got import request for {0} bytes", msg.Size);
var watch = Stopwatch.StartNew();
var count = 0;
var size = 0;
var blob = _config.GetPageBlob(msg.StagingLocation);
_manager.AppendEventsToStore(msg.StoreId, msg.StreamId, EnumerateStaging(blob, msg.Size).Select(bytes =>
{
count += 1;
size += bytes.Length;
return(bytes);
}));
var totalSeconds = watch.Elapsed.TotalSeconds;
var speed = size / totalSeconds;
Log.Info("Import {0} in {1}sec: {2} m/s or {3}", count, Math.Round(totalSeconds, 4), Math.Round(count / totalSeconds), FormatEvil.SpeedInBytes(speed));
msg.Envelope(new ClientMessage.ImportEventsCompleted());
ThreadPool.QueueUserWorkItem(state => CleanupBlob(blob));
}
public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
int threadCount = 5;
int batchSize = 10000;
int repeatForEachThread = 1;
int msgSize = 10;
string streamId = "batch";
if (args.Length > 0)
{
int.TryParse(args[0], out threadCount);
}
if (args.Length > 1)
{
int.TryParse(args[1], out batchSize);
}
if (args.Length > 2)
{
int.TryParse(args[2], out repeatForEachThread);
}
if (args.Length > 3)
{
int.TryParse(args[3], out msgSize);
}
long totalMs = 0;
var bytes = new byte[msgSize];
new RNGCryptoServiceProvider().GetBytes(bytes);
var threads = new List <Task>();
for (int t = 0; t < threadCount; t++)
{
var task = Task.Factory.StartNew(() =>
{
var watch = Stopwatch.StartNew();
for (int i = 0; i < repeatForEachThread; i++)
{
context.Client.EventStores.WriteEventsInLargeBatch(streamId, Enumerable.Repeat((bytes), batchSize));
}
Interlocked.Add(ref totalMs, watch.ElapsedMilliseconds);
}, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness);
threads.Add(task);
}
Task.WaitAll(threads.ToArray());
//context.Completed();
// througput
var totalMessages = threadCount * repeatForEachThread * batchSize;
var totalBytes = bytes.Length * totalMessages;
var key = string.Format("WB-{0}-{1}-{2}-{3}", threadCount, repeatForEachThread, batchSize, bytes.Length);
var bytesPerSec = (totalBytes * 1000D / totalMs);
var msgPerSec = (1000D * totalMessages / totalMs);
context.Log.Debug("Throughput: {0} or {1}", FormatEvil.SpeedInBytes(bytesPerSec), (int)msgPerSec);
context.Log.Debug("Average latency {0}ms", (int)totalMs / threadCount);
context.Log.Debug("Sent total {0} with {1}msg in {2}ms", FormatEvil.SizeInBytes(totalBytes), totalMessages, totalMs);
PerfUtils.LogTeamCityGraphData(key + "-bytesPerSec", (int)bytesPerSec);
PerfUtils.LogTeamCityGraphData(key + "-msgPerSec", (int)msgPerSec);
return(true);
}
