public void MultiCompositionTest4()
{
Factory factory = new Factory();
IPersonCollection personCollection = factory.CreatePersonCollection();
personCollection.Add(factory.CreatePerson("John", "Smith"));
personCollection.CollectionChanged += (sender, args) => Trace.WriteLine($"sender: {sender}, type: {args.Action}");
}
public MultiCompositionViewModel()
{
IPersonCollection personCollection = factory.CreatePersonCollection();
personCollection.CollectionChanged += (sender, args) => Trace.WriteLine($"sender: {sender}, type: {args.Action}");
Items = personCollection;
AddCommand = new Command(() => personCollection.Add(CreateNewPerson()));
RemoveCommand = new Command(() => personCollection.Remove(Selected));
ResetCommand = new Command(() => personCollection.Clear());
}
public void MultiCompositionTest1()
{
Factory factory = new Factory();
IPersonCollection personCollection = factory.CreatePersonCollection();
personCollection.Add(factory.CreatePerson("John", "Smith"));
CollectionAssert.AreEquivalent(
new[] { "John Smith" },
personCollection.Select(person => $"{person.FirstName} {person.LastName}").ToArray());
}
private void AddPersons(int count, IPersonCollection persons)
{
for (int i = 0; i < count; i++)
{
persons.AddPerson(
email: "pesho" + i + "@gmail" + (i % 100) + ".com",
name: "Pesho" + (i % 100),
age: i % 100,
town: "Sofia" + (i % 100));
}
}
public void MultiCompositionTest2()
{
Factory factory = new Factory();
IPersonCollection personCollection = factory.CreatePersonCollection();
List <IPerson> added = new List <IPerson>();
personCollection.CollectionChanged += (_, args) => added.AddRange(args.NewItems.OfType <IPerson>());
IPerson person = factory.CreatePerson("John", "Smith");
personCollection.Add(person);
Assert.AreEqual(person, added.Single());
}
public void BeginGeneration(IPersonCollection cohort, DirectoryInfo target)
{
//already running
if (sizeAtBeginGeneration != -1)
{
return;
}
sizeAtBeginGeneration = GetSize();
var fi = new FileInfo(Path.Combine(target.FullName, Generator.GetType().Name + ".csv"));
Thread = new Thread(() => Generator.GenerateTestDataFile(cohort, fi, sizeAtBeginGeneration));
Thread.Start();
}
private static DicomDataGenerator GetDataGenerator(ProgramOptions opts, IPersonCollection identifiers, Random r, out DirectoryInfo dir)
{
dir = Directory.CreateDirectory(opts.OutputDirectory);
//Generate the dicom files (of the modalities that the user requested)
string[] modalities = !string.IsNullOrWhiteSpace(opts.Modalities)? opts.Modalities.Split(",") :new string[0];
return(new DicomDataGenerator(r, dir, modalities)
{
NoPixels = opts.NoPixels,
Anonymise = opts.Anonymise,
Layout = opts.Layout,
MaximumImages = opts.MaximumImages,
Csv = opts.csv,
});
}
public IPersonCollection CreatePersonCollection()
{
List <IPerson> persons = new();
CollectionChangedImplementation <IPerson> collectionChanged = new(person => persons.IndexOf(person));
TypeDefinition <IPersonCollection> collectionTypeDefinition =
TypeDefinition <IPersonCollection> .Create(
new LinkedObjects(
new MappedMethod("Remove", collectionChanged, nameof(collectionChanged.PreRemove)),
persons,
collectionChanged));
IPersonCollection personCollection = collectionTypeDefinition.CreateNew();
IEventTrigger trigger = new EventTrigger(personCollection, nameof(IPersonCollection.CollectionChanged));
collectionChanged.CollectionChanged += (sender, args) => trigger.Notify(sender, args);
return(personCollection);
}
//[TestMethod]
public void MultiCompositionTest5()
{
Factory factory = new();
IPersonCollection personCollection = factory.CreatePersonCollection();
personCollection.Add(factory.CreatePerson("John", "Smith"));
IPerson person = factory.CreatePerson("Alice", "Smith");
personCollection.Add(person);
personCollection.Add(factory.CreatePerson("Johnny", "Smith"));
personCollection.Add(factory.CreatePerson("JJ", "Smith"));
object instance = null;
personCollection.CollectionChanged += (currentInstance, _) => instance = currentInstance;
personCollection.Remove(person);
Assert.IsNotNull(instance);
Assert.AreEqual(personCollection, instance);
}
public void MultiCompositionTest3()
{
Factory factory = new Factory();
IPersonCollection personCollection = factory.CreatePersonCollection();
personCollection.Add(factory.CreatePerson("John", "Smith"));
IPerson person = factory.CreatePerson("Alice", "Smith");
personCollection.Add(person);
personCollection.Add(factory.CreatePerson("Johnny", "Smith"));
personCollection.Add(factory.CreatePerson("JJ", "Smith"));
int removedIndex = -1;
personCollection.CollectionChanged += (_, args) => removedIndex = args.OldStartingIndex;
personCollection.Remove(person);
Assert.IsFalse(personCollection.Any(person1 => person1.FirstName == "Alice"));
Assert.AreEqual(1, removedIndex);
}
public IPersonCollection CreatePersonCollection()
{
List <IPerson> persons = new List <IPerson>();
CollectionChangedImplementation <IPerson> collectionChanged =
new CollectionChangedImplementation <IPerson>(person => persons.IndexOf(person));
TypeDefinition <IPersonCollection> typeDefinition =
new TypeDefinition <IPersonCollection>(
new LinkedObjects(
new MappedMethod("Remove", collectionChanged, nameof(collectionChanged.PreRemove)),
persons,
collectionChanged));
IEventTrigger trigger = null;
typeDefinition.RegisterEvent(
nameof(IPersonCollection.CollectionChanged),
eventTrigger => trigger = eventTrigger);
IPersonCollection personCollection = typeDefinition.Create();
collectionChanged.CollectionChanged += (sender, args) => trigger.Notify(sender, args);
return(personCollection);
}
public CreatePersonConsumer(IPersonCollection personCollection, IPersonOperations operations)
{
_personCollection = personCollection;
_operations = operations;
}
private static void RunBatch(IPersonCollection identifiers, ProgramOptions opts, Random r, DataTable[] batches, IBulkCopy[] uploaders)
{
Stopwatch swGeneration = new Stopwatch();
Stopwatch swReading = new Stopwatch();
Stopwatch swUploading = new Stopwatch();
try
{
using (var dicomGenerator = GetDataGenerator(opts, identifiers, r, out _))
{
for (int i = 0; i < opts.NumberOfStudies; i++)
{
swGeneration.Start();
var p = identifiers.People[r.Next(identifiers.People.Length)];
var ds = dicomGenerator.GenerateStudyImages(p, out Study s);
swGeneration.Stop();
foreach (DicomDataset dataset in ds)
{
var rows = new DataRow[batches.Length];
for (int j = 0; j < batches.Length; j++)
{
rows[j] = batches[j].NewRow();
}
swReading.Start();
foreach (DicomItem item in dataset)
{
var column = DicomTypeTranslaterReader.GetColumnNameForTag(item.Tag, false);
var value = DicomTypeTranslater.Flatten(DicomTypeTranslaterReader.GetCSharpValue(dataset, item));
foreach (DataRow row in rows)
{
if (row.Table.Columns.Contains(column))
{
row[column] = value ?? DBNull.Value;
}
}
}
for (int j = 0; j < batches.Length; j++)
{
batches[j].Rows.Add(rows[j]);
}
swReading.Stop();
}
//every 100 and last batch
if (i % 100 == 0 || i == opts.NumberOfStudies - 1)
{
swUploading.Start();
for (var j = 0; j < uploaders.Length; j++)
{
uploaders[j].Upload(batches[j]);
batches[j].Rows.Clear();
}
swUploading.Stop();
Console.WriteLine($"{DateTime.Now} Done {i} studies");
}
}
}
}
finally
{
for (var i = 0; i < uploaders.Length; i++)
{
uploaders[i].Dispose();
batches[i].Dispose();
}
}
Console.WriteLine("Total time Generating Dicoms:" + swGeneration.Elapsed);
Console.WriteLine("Total time Reading Dicoms:" + swReading.Elapsed);
Console.WriteLine("Total time Uploading Records:" + swUploading.Elapsed);
}
private static int RunDatabaseTarget(TargetDatabase configDatabase, ProgramOptions opts)
{
var batchSize = Math.Max(1, configDatabase.Batches);
//if we are going into a database we definitely do not need pixels!
if (opts.NoPixels == false)
{
opts.NoPixels = true;
}
Stopwatch swTotal = new Stopwatch();
swTotal.Start();
string neverDistinct = "SOPInstanceUID";
if (!File.Exists(configDatabase.Template))
{
Console.WriteLine($"Listed template file '{configDatabase.Template}' does not exist");
return(-1);
}
ImageTableTemplateCollection template;
try
{
template = ImageTableTemplateCollection.LoadFrom(File.ReadAllText(configDatabase.Template));
}
catch (Exception e)
{
Console.WriteLine($"Error reading yaml from '{configDatabase.Template}'");
Console.WriteLine(e.ToString());
return(-2);
}
ImplementationManager.Load <MySqlImplementation>();
ImplementationManager.Load <PostgreSqlImplementation>();
ImplementationManager.Load <OracleImplementation>();
ImplementationManager.Load <MicrosoftSQLImplementation>();
var server = new DiscoveredServer(configDatabase.ConnectionString, configDatabase.DatabaseType);
try
{
server.TestConnection();
}
catch (Exception e)
{
Console.WriteLine($"Could not reach target server '{server.Name}'");
Console.WriteLine(e);
return(-2);
}
var db = server.ExpectDatabase(configDatabase.DatabaseName);
if (!db.Exists())
{
Console.WriteLine($"Creating Database '{db.GetRuntimeName()}'");
db.Create();
Console.WriteLine("Database Created");
}
else
{
Console.WriteLine($"Found Database '{db.GetRuntimeName()}'");
}
var creator = new ImagingTableCreation(db.Server.GetQuerySyntaxHelper());
Console.WriteLine($"Image template contained schemas for {template.Tables.Count} tables. Looking for existing tables..");
//setting up bulk inserters
DiscoveredTable[] tables = new DiscoveredTable[template.Tables.Count];
DataTable[][] batches = new DataTable[batchSize][];
for (var i = 0; i < batches.Length; i++)
{
batches[i] = new DataTable[template.Tables.Count];
}
IBulkCopy[][] uploaders = new IBulkCopy[batchSize][];
for (int i = 0; i < uploaders.Length; i++)
{
uploaders[i] = new IBulkCopy[template.Tables.Count];
}
string[] pks = new string[template.Tables.Count];
for (var i = 0; i < template.Tables.Count; i++)
{
var tableSchema = template.Tables[i];
var tbl = db.ExpectTable(tableSchema.TableName);
tables[i] = tbl;
if (configDatabase.MakeDistinct)
{
var col = tableSchema.Columns.Where(c => c.IsPrimaryKey).ToArray();
if (col.Length > 1)
{
Console.WriteLine("MakeDistinct only works with single column primary keys e.g. StudyInstanceUID / SeriesInstanceUID");
}
pks[i] = col.SingleOrDefault()?.ColumnName;
if (pks[i] != null)
{
//if it is sop instance uid then we shouldn't be trying to deduplicate
if (string.Equals(pks[i], neverDistinct, StringComparison.CurrentCultureIgnoreCase))
{
pks[i] = null;
}
else
{
//we will make this a primary key later on
col.Single().IsPrimaryKey = false;
Console.WriteLine($"MakeDistinct will apply to '{pks[i]}' on '{tbl.GetFullyQualifiedName()}'");
}
}
}
bool create = true;
if (tbl.Exists())
{
if (configDatabase.DropTables)
{
Console.WriteLine($"Dropping existing table '{tbl.GetFullyQualifiedName()}'");
tbl.Drop();
}
else
{
Console.WriteLine($"Table '{tbl.GetFullyQualifiedName()}' already existed (so will not be created)");
create = false;
}
}
if (create)
{
Console.WriteLine($"About to create '{tbl.GetFullyQualifiedName()}'");
creator.CreateTable(tbl, tableSchema);
Console.WriteLine($"Successfully created create '{tbl.GetFullyQualifiedName()}'");
}
Console.WriteLine($"Creating uploader for '{tbl.GetRuntimeName()}''");
for (int j = 0; j < batchSize; j++)
{
//fetch schema
var dt = tbl.GetDataTable();
dt.Rows.Clear();
batches[j][i] = dt;
uploaders[j][i] = tbl.BeginBulkInsert();
}
}
var tasks = new Task[batchSize];
IPersonCollection identifiers = GetPeople(opts, out Random r);
for (int i = 0; i < batchSize; i++)
{
var batch = i;
tasks[i] = new Task(() => // lgtm[cs/local-not-disposed]
{
RunBatch(identifiers, opts, r, batches[batch], uploaders[batch]);
});
tasks[i].Start();
}
Task.WaitAll(tasks);
swTotal.Stop();
for (var i = 0; i < tables.Length; i++)
{
if (pks[i] == null)
{
continue;
}
Console.WriteLine($"{DateTime.Now} Making table '{tables[i]}' distinct (this may take a long time)");
var tbl = tables[i];
tbl.MakeDistinct(500000000);
Console.WriteLine($"{DateTime.Now} Creating primary key on '{tables[i]}' of '{pks[i]}'");
tbl.CreatePrimaryKey(500000000, tbl.DiscoverColumn(pks[i]));
}
Console.WriteLine("Final Row Counts:");
foreach (DiscoveredTable t in tables)
{
Console.WriteLine($"{t.GetFullyQualifiedName()}: {t.GetRowCount():0,0}");
}
Console.WriteLine("Total Running Time:" + swTotal.Elapsed);
return(0);
}
public void TestInitialize()
{
//this.persons = new PersonCollectionSlow();
this.persons = new PersonCollection();
}
private static void RunOptionsAndReturnExitCode(ProgramOptions opts)
{
if (opts.NumberOfPatients <= 0)
{
opts.NumberOfPatients = 500;
}
if (opts.NumberOfStudies <= 0)
{
opts.NumberOfStudies = 2000;
}
if (File.Exists(ConfigFile))
{
Config config;
try
{
var d = new Deserializer();
config = d.Deserialize <Config>(File.ReadAllText(ConfigFile));
}
catch (Exception e)
{
Console.WriteLine($"Error deserializing '{ConfigFile}'");
Console.Write(e.ToString());
_returnCode = -1;
return;
}
if (config.Database != null)
{
try
{
_returnCode = RunDatabaseTarget(config.Database, opts);
return;
}
catch (Exception e)
{
Console.WriteLine(e);
_returnCode = 3;
return;
}
}
}
try
{
IPersonCollection identifiers = GetPeople(opts, out Random r);
using (var dicomGenerator = GetDataGenerator(opts, identifiers, r, out DirectoryInfo dir))
{
Console.WriteLine($"{DateTime.Now} Starting file generation (to {dir.FullName})");
var targetFile = new FileInfo(Path.Combine(dir.FullName, "DicomFiles.csv"));
dicomGenerator.GenerateTestDataFile(identifiers, targetFile, opts.NumberOfStudies);
}
}
catch (Exception e)
{
Console.WriteLine(e);
_returnCode = 2;
return;
}
Console.WriteLine($"{DateTime.Now} Finished");
_returnCode = 0;
}
public void TestInitialize()
{
this.persons = new PersonCollectionSlow();
//this.persons = new PersonCollection();
}
public void TestInitilize()
{
this.persons = new PersonCollection();
}
