private static void AppendNewDocument(
StringBuilder builder,
string indexName,
List <ReportFile> files,
ReportType reportType,
List <CpasUser> users,
List <CpasUser> substitutionable,
DateTime baseDate,
object docId,
ref int reportsSelected)
{
CpasUser substitution = null;
if (random.Next(100) == 1)
{
substitution = coverageUtils.GetOne(substitutionable);
}
List <ReportFile> reports = coverageUtils
.GetUpTo(4, files);
reportsSelected += reports.Count;
builder
.Append(JsonConvert.SerializeObject(new
{
index = new
{
_index = indexName,
_id = docId
}
}))
.AppendLine()
.Append(
JsonConvert.SerializeObject(new
{
user = coverageUtils.GetOne(users),
substitution,
downloaded_at = GetDateForReport(baseDate),
from = GetRandomLocation(),
reports,
report_type = reportType
}))
.AppendLine();
}
private static async Task CreateDocuments(
int documentsToCreate,
int idOffset,
int documentsBatchSize,
string indexName,
List <DummyUser> users,
Stopwatch watches)
{
Console.WriteLine("Creating demo documents...");
StringBuilder builder = new StringBuilder();
for (int i = 0; i < (int)Math.Ceiling(documentsToCreate / (decimal)documentsBatchSize); i++)
{
Console.WriteLine($"Batch {i} started");
watches.Restart();
for (
int j = (i * documentsBatchSize) + idOffset;
j < ((i + 1) * documentsBatchSize) + idOffset; // && j < documentsToCreate + idOffset
j++
)
{
builder
.Append(JsonConvert.SerializeObject(new
{
index = new
{
_index = indexName,
_id = j
}
}))
.AppendLine()
.Append(
JsonConvert.SerializeObject(new
{
title = $"Document_{j:0000}.txt",
author = users[random.Next(users.Count)].Username,
created = DateTime.Now,
content = "Document's content",
roles = coverageUtils.GetUpTo(3, CoverageConfig.DocumentPermissions),
mainarea = coverageUtils.GetRandom(CoverageConfig.MainAreaCoverage),
subarea = coverageUtils.GetRandom(CoverageConfig.SubAreaCoverage),
product = coverageUtils.GetRandom(CoverageConfig.ProductsCoverage),
placement = coverageUtils.GetRandom(CoverageConfig.PlacementCoverage),
users = new[]
{
coverageUtils.GetOne(users).Username,
coverageUtils.GetOne(users).Username,
coverageUtils.GetOne(users).Username
},
organization = coverageUtils.GetUpTo(3, CoverageConfig.OrganizationCoverage)
}))
.AppendLine();
}
Console.WriteLine($"Generating NDJson string for batch took: {watches.ElapsedMilliseconds}ms");
Console.WriteLine("Starting sending data to Elastic");
StringContent content = new StringContent(builder.ToString());
watches.Restart();
content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Json);
await $"{Config.ElasticSearchAddress}/{indexName}/_bulk"
.WithBasicAuth("admin", "admin")
.PostAsync(
content
);
Console.WriteLine($"Sending data took: {watches.ElapsedMilliseconds}ms");
builder.Clear();
}
}
public static async Task Main(string[] args)
{
FlurlHttp.ConfigureClient(Config.ElasticSearchAddress, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
FlurlHttp.GlobalSettings.BeforeCall = call =>
{
Console.WriteLine($"Before: {call.Request.RequestUri}");
};
FlurlHttp.GlobalSettings.BeforeCall = call =>
{
Console.WriteLine($"After: {call.Request.RequestUri}");
};
#region Get input
string indexName;
int usersToCreate;
Console.Write("Use default values? [y/...]: ");
if (Console.ReadLine() != "y")
{
Console.Write("Index name: ");
indexName = Console.ReadLine();
usersToCreate = ConsoleUtils.PromptValue("Users to create: ");
}
else
{
indexName = "cz_index";
usersToCreate = 1500;
}
#endregion
Tuple <List <DummyUser>, long> users = await dummyUtils.CreateUsers(
usersToCreate, 0, short.MaxValue, false, Stopwatch.StartNew());
Console.WriteLine("Getting all documents");
HttpResponseMessage documentsResponse = await $"{Config.ElasticSearchAddress}/{indexName}/_search"
.PostJsonAsync(
new
{
_source = new dynamic[0],
size = 10000
});
Console.WriteLine($"Parsing all documents");
JObject parsedDocs = JObject.Parse(await documentsResponse.Content.ReadAsStringAsync());
Console.WriteLine($"{parsedDocs["hits"]["total"]["value"].Value<int>()} documents parsed");
List <Task> requests = new List <Task>(
parsedDocs["hits"]["total"]["value"].Value <int>()
);
Console.WriteLine("Sending update requests");
foreach (JToken document in parsedDocs["hits"]["hits"].AsJEnumerable())
{
requests.Add($"{Config.ElasticSearchAddress}/{indexName}/_update/{document["_id"]}"
.PostJsonAsync(new
{
doc = new
{
mainarea = coverageUtils.GetRandom(CoverageConfig.MainAreaCoverage),
subarea = coverageUtils.GetRandom(CoverageConfig.SubAreaCoverage),
product = coverageUtils.GetRandom(CoverageConfig.ProductsCoverage),
placement = coverageUtils.GetRandom(CoverageConfig.PlacementCoverage),
users = new[]
{
coverageUtils.GetOne(users.Item1),
coverageUtils.GetOne(users.Item1),
coverageUtils.GetOne(users.Item1)
}
}
}));
}
Console.WriteLine("Waiting for unfinished requests");
Task.WaitAll(requests.ToArray());
Console.WriteLine("Done!");
}