public static async Task UploadResult(ArtilleryResult artilleryResult, string serverurl, string username, string password, Dictionary <string, string> extraFields)
{
var allrequests = new List <ElasticBulkDocument>();
foreach (var request in artilleryResult.Requests)
{
dynamic jobject = new JObject
{
["LoadtestID"] = artilleryResult.LoadtestID,
["@timestamp"] = request.StartTime.ToString("yyyy-MM-ddTHH:mm:ss.fff"),
["LatencyNS"] = request.Latency,
["HttpResult"] = request.HttpResult,
["RebasedTimestamp"] = request.StartTime.AddMilliseconds(artilleryResult.Diff_ms).ToString("yyyy-MM-ddTHH:mm:ss.fff")
};
foreach (var field in extraFields)
{
jobject[field.Key] = field.Value;
}
var bulkDocument = new ElasticBulkDocument
{
Index = $"artillery-{request.StartTime:yyyy}.{request.StartTime:MM}",
Id = GetHashString(jobject.ToString()),
Document = jobject
};
allrequests.Add(bulkDocument);
}
Log($"Request count: {allrequests.Count}");
await Elastic.PutIntoIndex(serverurl, username, password, allrequests.ToArray());
}
bool UnitTest2()
{
var dic = new Dictionary <int, int>
{
[11] = 2,
[3] = 3
};
var results = new List <int>();
for (int i = 0; i < 6; i++)
{
results.Add(ArtilleryResult.PopSmallestKey(dic));
}
var compare = new[] { 3, 3, 3, 11, 11, 0 };
bool error = false;
for (int i = 0; i < 6; i++)
{
if (results[i] != compare[i])
{
error = true;
Log($"ERROR: Expected {compare[i]}, was {results[i]}");
}
else
{
Log($"OK: Expected {compare[i]}, was {results[i]}");
}
}
return(error);
}
public static ArtilleryResult ParseFile(string filename, long rebasestarttime)
{
string content = File.ReadAllText(filename);
dynamic document = JObject.Parse(content);
string loadtestID = GetHashString(document.ToString());
var result = new ArtilleryResult
{
LoadtestID = loadtestID
};
result.EarliestStartTime = GetEarliestStartTime(document);
result.LastEndTime = GetLastEndTime(document);
if (result.EarliestStartTime > result.LastEndTime)
{
Log("Couldn't find any latencies.");
return(null);
}
result.Diff_ms = GetDiff((long)result.EarliestStartTime.TimeOfDay.TotalMilliseconds, rebasestarttime * 1000);
Log($"EarliestStartTime: {result.EarliestStartTime:yyyy-MM-dd HH:mm:ss.fff}, LastEndTime: {result.LastEndTime:yyyy-MM-dd HH:mm:ss.fff}, Diff_ms: {result.Diff_ms}");
result.Requests = GetRequests(document);
Log($"Got {result.Requests.Length} requests/latencies.");
return(result);
}
bool UnitTest1()
{
long[,] difftestshours = new long[, ]
{
{ 0, 0, 0 },
{ 0, 1, 1 },
{ 0, 11, 11 },
{ 0, 12, 12 },
{ 0, 13, -11 },
{ 0, 23, -1 },
{ 23, 0, 1 },
{ 23, 1, 2 },
{ 23, 11, 12 },
{ 23, 12, -11 },
{ 23, 13, -10 },
{ 23, 23, 0 },
{ 3, 4, 1 },
{ 3, 2, -1 },
{ 3, 23, -4 },
{ 3, 14, 11 },
{ 3, 15, 12 },
{ 3, 16, -11 }
};
Log($"{difftestshours.Length}");
for (int i = 0; i < difftestshours.Length / 3; i++)
{
long timeSinceMidnightMs = difftestshours[i, 0];
long desiredTimeOfDayMs = difftestshours[i, 1];
long timediffms = difftestshours[i, 2];
long resultms = ArtilleryResult.GetDiff(timeSinceMidnightMs * 3600 * 1000, desiredTimeOfDayMs * 3600 * 1000);
if (resultms != timediffms * 3600 * 1000)
{
Log($"ERROR: {timeSinceMidnightMs} {desiredTimeOfDayMs} was: {resultms / 3600 / 1000} should be: {timediffms}");
}
else
{
Log($"OK: {timeSinceMidnightMs} {desiredTimeOfDayMs} was {resultms / 3600 / 1000}");
}
}
return(true);
}
static async Task <int> Main(string[] args)
{
int stopParse = Array.FindIndex(args, a => a == "--");
List <string> parsedArgs = stopParse > 0 ? args.Take(stopParse).ToList() : args.ToList();
if (parsedArgs.Contains("-unittest"))
{
return(new UnitTests().RunUnitTests() ? 0 : 1);
}
var extraFields = ExtractExtraFields(parsedArgs);
if (parsedArgs.Count != 5)
{
Log(
@"Usage: ArtilleryToElastic.exe [-f <name> <value>] <filename> <serverurl> <username> <password> <rebasetime>
This tool imports an Artillery result file into Elasticsearch, and optionally copies other indices.
All imported/copied documents will have an added field that contains a rebased timestamp,
this is useful to make nighly loadtests appear to start at exactly the same time each day,
this makes everything easy to visualize, which is the real purpose of this tool.
-f: Optional extra fields that will be added to every json document.
May be specified multiple times to add multiple name/value pairs.
filename: Artillery result input file (json).
serverurl: Target elasticsearch base url.
username: Target elasticsearch username.
password: Target elasticsearch password.
rebasetime: Start time (HH:mm:ss) that timestamps should be rebased on.
Environment variables, to copy extra documents with an applied Rebased[Timestamp] field. To copy multiple
indices, prefix and/or suffixed the variable names.
ElasticSourceServerurl: Elasticsearch base url.
ElasticSourceUsername: Elasticsearch username.
ElasticSourcePassword: Elasticsearch password.
ElasticSourceIndex: Elasticsearch source index.
ElasticTargetIndex: Elasticsearch target index. Optional. May end with date pattern, yyyy.mm.dd, yyyy.mm or yyyy.
ElasticTimestampField: Elasticsearch timestamp field. A Rebased[Timestamp] field will be added.
ElasticFilterField: Filter to reduce number of copied documents, field name. Optional.
ElasticFilterValue: Filter to reduce number of copied documents, field value. Optional.");
return(1);
}
string filename = parsedArgs[0];
string serverurl = parsedArgs[1];
string username = parsedArgs[2];
string password = parsedArgs[3];
var elasticCopySources = GetElasticCopySources();
if (!TryParseTime(parsedArgs[4], out long rebasestarttime))
{
Log("Invalid rebasetime format, use HH:mm:ss pattern.");
return(1);
}
ArtilleryResult result = ArtilleryResult.ParseFile(filename, rebasestarttime);
await ArtilleryToElastic.UploadResult(result, serverurl, username, password, extraFields);
foreach (var elasticSource in elasticCopySources)
{
await CopyElasticLogs.CopyDocuments(elasticSource,
serverurl, username, password, elasticSource.TargetIndex, result.EarliestStartTime.AddMinutes(-5), result.LastEndTime.AddMinutes(5), result.Diff_ms,
extraFields);
}
return(0);
}
