protected override void Given()
{
_markerString = GetType().FullName;
_generatorSetup = new GeneratorSetup();
_generatorSetup.DefaultFlushStrategy = new LimitedCapacityFlushStrategy(10);
_generatorSetup.DefaultPersistentStorages.Add(
new EntityFrameworkCorePersistentStorage(() => new SampleDbContext()));
_generatorSetup.RegisterEntity <Category>()
.SetTargetCount(25)
.SetGenerator(ctx => new Category()
{
Name = $"Category #{ctx.CurrentCount}",
MarkerText = _markerString
});
_generatorSetup.RegisterEntity <Post>()
.SetInsertToPersistentStorageBeforeUse(true)
.SetTargetCount(100)
.SetGenerator(ctx => new Post()
{
MarkerText = _markerString
});
_generatorSetup.RegisterEntity <Comment>()
.SetTargetCount(50)
.SetGenerator <Category, Post>((ctx, category, post) => new Comment()
{
PostId = post.Id,
CommentText = $"Comment in category: {category.Name}",
MarkerText = _markerString
});
}
static void Main(string[] args)
{
//Setup
var setup = new GeneratorSetup();
setup.RegisterEntity <Buyer>()
.SetGenerator(GenerateBuyer)
.SetPersistentStorage(new CsvPersistentStorage("Set-Buyers.csv"))
.SetTargetCount(100);
setup.RegisterEntity <Supplier>()
.SetGenerator(GenerateSupplier)
.SetPersistentStorage(new CsvPersistentStorage("Set-Suppliers.csv"))
.SetTargetCount(100);
setup.RegisterEntity <PurchaseOrder>()
.SetGenerator <Supplier, Buyer>(GeneratePurchaseOrder)
.AddMultiModifier <Supplier, Buyer>(ModifyPurchaseOrders)
.SetPersistentStorage(new CsvPersistentStorage("Set-PurchaseOrders.csv"))
.SetSpreadStrategy(new CartesianProductSpreadStrategy())
.SetTargetCount(100000);
//Generate
setup.ProgressChanged += PrintProgress;
setup.Generate();
Console.WriteLine("Completed");
Console.ReadKey();
}
private StreamReader GenerateTest(GeneratorSetup generatorSetup)
{
StringBuilder outputBuilder = new StringBuilder();
string output = "";
using (AutoResetEvent outputWaitHandle = new AutoResetEvent(false))
using (AutoResetEvent errorWaitHandle = new AutoResetEvent(false))
{
using (Process process = new Process())
{
process.StartInfo.FileName = "../../knapgen.exe";
process.StartInfo.UseShellExecute = false;
process.EnableRaisingEvents = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.RedirectStandardError = true;
process.StartInfo.Arguments =
$"-I {generatorSetup.StartingId}" +
$" -n {generatorSetup.ItemCount}" +
$" -N {generatorSetup.InstanceCount}" +
$" -m {generatorSetup.RatioCapacityToWeightSum}" +
$" -W {generatorSetup.MaxWeight}" +
$" -C {generatorSetup.MaxCost}" +
$" -k {generatorSetup.Exponent}" +
$" -d {(generatorSetup.EqualThings ? 0 : (generatorSetup.LessThings ? -1 : 1))}";
try
{
process.OutputDataReceived += (sender, e) =>
{
if (e.Data != null)
{
outputBuilder.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (sender, e) =>
{
if (e.Data != null)
{
// outputBuilder.AppendLine(e.Data);
}
};
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
process.WaitForExit();
output = outputBuilder.ToString();
}
finally
{
outputWaitHandle.WaitOne(1);
errorWaitHandle.WaitOne(1);
}
}
}
return new StreamReader(GenerateStreamFromString(output));
}
//init
public ReleaseFromTempStorageCommand(EntityContext entityContext, GeneratorSetup setup,
IFlushCandidatesRegistry flushCandidatesRegistry)
{
_entityContext = entityContext;
_setup = setup;
_flushCandidatesRegistry = flushCandidatesRegistry;
}
//init
public CheckFlushRequiredCommand(EntityContext entityContext, GeneratorSetup setup,
IFlushCandidatesRegistry flushCandidatesRegistry)
{
_entityContext = entityContext;
_setup = setup;
_flushCandidatesRegistry = flushCandidatesRegistry;
}
//init
public CompleteRequiredQueueBuilder(GeneratorSetup generatorSetup,
Dictionary <Type, EntityContext> entityContexts, INextNodeFinder nextNodeFinder)
{
_generatorSetup = generatorSetup;
_entityContexts = entityContexts;
_nextNodeFinder = nextNodeFinder;
}
//init
public GenerateEntitiesCommand(EntityContext entityContext, GeneratorSetup setup,
Dictionary <Type, EntityContext> entityContexts)
{
EntityContext = entityContext;
_setup = setup;
_entityContexts = entityContexts;
}
//init
public FlushCommand(EntityContext entityContext, GeneratorSetup setup,
IFlushCandidatesRegistry flushCandidatesRegistry)
{
EntityContext = entityContext;
_setup = setup;
_flushCandidatesRegistry = flushCandidatesRegistry;
}
//register generators
public virtual void RegisterEntities(GeneratorSetup setup, SpecsDbContext dbContext)
{
IMongoDatabase _database = dbContext.SignalBounces.Database;
setup.RegisterEntity <SignalDispatch <ObjectId> >()
.SetGenerator(GenerateDispatches)
.SetPersistentStorage(new MongoDbPersistentStorage(_database, dbContext.SignalDispatches.CollectionNamespace.CollectionName));
}
//init
public CompleteNextNodeFinder(GeneratorSetup generatorSetup,
IFlushCandidatesRegistry flushCandidatesRegistry,
IProgressState progressState)
{
_generatorSetup = generatorSetup;
_flushCandidatesRegistry = flushCandidatesRegistry;
_progressState = progressState;
}
//init
public CompleteFlushCandidatesRegistry(GeneratorSetup generatorSetup,
Dictionary <Type, EntityContext> entityContexts, IProgressState progressState)
{
_flushCandidates = new HashSet <EntityContext>();
_generatorSetup = generatorSetup;
_entityContexts = entityContexts;
_progressState = progressState;
}
public TestResult(double averageError, double maxError, double averageTime, double maxTime, int n, GeneratorSetup generatorSetup)
{
AverageError = averageError;
MaxError = maxError;
AverageTime = averageTime;
MaxTime = maxTime;
N = n;
GeneratorSetup = generatorSetup;
}
private static void GenerateExponent(ICollection<GeneratorSetup> list)
{
for (var f = 0.01f; f <= 3.0f; f += 0.05f)
{
GeneratorSetup gen = new GeneratorSetup();
gen.GeneratorName = MethodBase.GetCurrentMethod().Name;
gen.Exponent = f;
list.Add(gen);
}
}
private static void GenerateMaxWeight(ICollection<GeneratorSetup> list)
{
for (var i = 100; i <= 5100; i += 500)
{
GeneratorSetup gen = new GeneratorSetup();
gen.GeneratorName = MethodBase.GetCurrentMethod().Name;
gen.MaxWeight = i;
list.Add(gen);
}
}
protected virtual IEntityDescription GetEntityDescription(GeneratorSetup setup, Type entityType)
{
bool isEntityRegistered = setup.EntityDescriptions.ContainsKey(entityType);
if (!isEntityRegistered || setup.EntityDescriptions[entityType] == null)
{
throw new KeyNotFoundException($"Entity type [{entityType.FullName}] is not registered.");
}
return(setup.EntityDescriptions[entityType]);
}
//init
public SubsetNodeFinder(List <Type> entitiesSubset, GeneratorSetup generatorSetup,
IFlushCandidatesRegistry flushCandidatesRegistry,
IProgressState progressState)
: base(generatorSetup, flushCandidatesRegistry, progressState)
{
if (entitiesSubset == null)
{
throw new ArgumentNullException(nameof(entitiesSubset));
}
_entitiesSubset = entitiesSubset;
}
//init
public SubsetFlushCandidatesRegistry(List <Type> entitiesSubset, GeneratorSetup generatorSetup,
Dictionary <Type, EntityContext> entityContexts, IProgressState orderProgress)
: base(generatorSetup, entityContexts, orderProgress)
{
if (entitiesSubset == null)
{
throw new ArgumentNullException(nameof(entitiesSubset));
}
_entitiesSubset = entitiesSubset;
}
protected virtual InMemoryStorage SetMemoryStorage(GeneratorSetup setup, Dictionary <Type, long> dataAmmounts)
{
var storage = new InMemoryStorage();
foreach (Type type in dataAmmounts.Keys)
{
IEntityDescription entityDescription = GetEntityDescription(setup, type);
entityDescription.PersistentStorages = entityDescription.PersistentStorages ?? new List <IPersistentStorage>();
entityDescription.PersistentStorages.Add(storage);
}
return(storage);
}
public override void Setup(GeneratorSetup generatorSetup,
Dictionary <Type, EntityContext> entityContexts)
{
base.Setup(generatorSetup, entityContexts);
ProgressState = new SubsetProgressState(_entitiesSubset, entityContexts);
_flushCandidatesRegistry = new SubsetFlushCandidatesRegistry(
_entitiesSubset, generatorSetup, entityContexts, ProgressState);
_nextNodeFinder = new SubsetNodeFinder(_entitiesSubset, _generatorSetup,
_flushCandidatesRegistry, ProgressState);
_requiredQueueBuilder = new CompleteRequiredQueueBuilder(
generatorSetup, entityContexts, _nextNodeFinder);
}
//Setup helpers
private CompleteSupervisor SetupCompleteOrderProvider(
List <IEntityDescription> descriptions)
{
var generatorSetup = new GeneratorSetup();
Dictionary <Type, IEntityDescription> dictDescriptions =
descriptions.ToDictionary(x => x.Type, x => x);
Dictionary <Type, EntityContext> contexts =
generatorSetup.SetupEntityContexts(dictDescriptions);
var target = new CompleteSupervisor();
target.Setup(generatorSetup, contexts);
return(target);
}
//methods
public override void RegisterEntities(GeneratorSetup setup, SpecsDbContext dbContext)
{
IMongoDatabase _database = dbContext.SignalBounces.Database;
setup.RegisterEntity <SubscriberWithMissingData>()
.SetGenerator(GenerateSubscriber);
setup.RegisterEntity <SpecsDeliveryTypeSettings>()
.SetMultiGenerator <SubscriberWithMissingData>(GenerateDeliveryTypes)
.SetPersistentStorage(new MongoDbPersistentStorage(_database, dbContext.SubscriberDeliveryTypeSettings.CollectionNamespace.CollectionName));
setup.RegisterEntity <SubscriberTopicSettings <ObjectId> >()
.SetMultiGenerator <SubscriberCategorySettings <ObjectId>, SubscriberWithMissingData>(GenerateTopics)
.SetPersistentStorage(new MongoDbPersistentStorage(_database, dbContext.SubscriberTopicSettings.CollectionNamespace.CollectionName));
}
private CompleteFlushCandidatesRegistry SetupCompleteFlushCandidatesRegistry(
Dictionary <Type, EntityContext> contexts)
{
var generatorSetup = new GeneratorSetup();
var progressState = new CompleteProgressState(contexts);
var target = new CompleteFlushCandidatesRegistry(generatorSetup,
contexts, progressState);
//Add flush candidates
target.CheckIsFlushRequired(contexts[typeof(Post)]);
target.CheckIsFlushRequired(contexts[typeof(Category)]);
target.CheckIsFlushRequired(contexts[typeof(Comment)]);
return(target);
}
//prepare settings
protected virtual void SetDataAmounts(GeneratorSetup setup, Dictionary <Type, long> dataAmmounts)
{
foreach (Type type in dataAmmounts.Keys)
{
IEntityDescription entityDescription = GetEntityDescription(setup, type);
long ammount = dataAmmounts[type];
PropertyInfo quantityPropInfo = entityDescription.GetType()
.GetProperty(nameof(entityDescription.QuantityProvider));
quantityPropInfo.SetValue(entityDescription, new StrictQuantityProvider(ammount));
PropertyInfo flushTriggerPropInfo = entityDescription.GetType()
.GetProperty(nameof(entityDescription.FlushTrigger));
flushTriggerPropInfo.SetValue(entityDescription, new LimitedCapacityFlushStrategy(ammount));
}
}
public void CheckGeneratorSetupComplete_WhenNotComplete()
{
//Prepare
var descriptions = new List <IEntityDescription>();
descriptions.Add(new EntityDescription <Post>()
.SetRequired(typeof(Category)));
Dictionary <Type, IEntityDescription> entityDescriptions =
descriptions.ToDictionary(x => x.Type, x => x);
//Invoke
var generatorSetup = new GeneratorSetup();
var target = new Validator(generatorSetup);
target.CheckGeneratorSetupComplete(entityDescriptions);
}
//generate entry point
public InMemoryStorage Generate(SpecsDbContext dbContext, IGeneratorData generatorData,
bool useMemoryStorage, Dictionary <Type, long> ammounts)
{
var setup = new GeneratorSetup();
generatorData.RegisterEntities(setup, dbContext);
SetDataAmounts(setup, ammounts);
InMemoryStorage storage = useMemoryStorage
? SetMemoryStorage(setup, ammounts)
: null;
setup.ProgressChanged += PrintProgress;
setup.Generate();
return(storage);
}
//init
public virtual void Setup(GeneratorSetup generatorSetup,
Dictionary <Type, EntityContext> entityContexts)
{
_commandsQueue = new ConcurrentQueue <ICommand>();
_entityContexts = entityContexts;
_generatorSetup = generatorSetup;
ProgressState = new CompleteProgressState(entityContexts);
_flushCandidatesRegistry = new CompleteFlushCandidatesRegistry(
generatorSetup, entityContexts, ProgressState);
_nextNodeFinder = new CompleteNextNodeFinder(
generatorSetup, _flushCandidatesRegistry, ProgressState);
_requiredQueueBuilder = new CompleteRequiredQueueBuilder(
generatorSetup, entityContexts, _nextNodeFinder);
}
public void CheckGeneratorSetupComplete_WhenComplete()
{
//Prepare
var generatorSetup = new GeneratorSetup();
generatorSetup.DefaultPersistentStorages.Add(new InMemoryStorage());
var descriptions = new List <IEntityDescription>();
descriptions.Add(new EntityDescription <Post>()
.SetTargetCount(5)
.SetGenerator(context => null));
Dictionary <Type, IEntityDescription> entityDescriptions =
descriptions.ToDictionary(x => x.Type, x => x);
//Invoke
var target = new Validator(generatorSetup);
target.CheckGeneratorSetupComplete(entityDescriptions);
}
public void CheckCircularDependencies_WhenNoCircleDependencies()
{
//Prepare
var descriptions = new List <IEntityDescription>();
descriptions.Add(new EntityDescription <Post>()
.SetRequired(typeof(Category)));
descriptions.Add(new EntityDescription <Comment>());
descriptions.Add(new EntityDescription <Category>()
.SetRequired(typeof(Comment)));
descriptions.Add(new EntityDescription <Attachment>()
.SetRequired(typeof(Post)));
Dictionary <Type, IEntityDescription> entityDescriptions =
descriptions.ToDictionary(x => x.Type, x => x);
//Invoke
var generatorSetup = new GeneratorSetup();
var target = new Validator(generatorSetup);
target.CheckCircularDependencies(entityDescriptions);
}
private static void GenerateLessEqualGreater(ICollection<GeneratorSetup> list)
{
var genL = new GeneratorSetup();
var genE = new GeneratorSetup();
var genG = new GeneratorSetup();
genL.LessThings = true;
genL.EqualThings = false;
genE.EqualThings = true;
genG.LessThings = false;
genG.EqualThings = false;
list.Add(genL);
list.Add(genE);
list.Add(genG);
genL.GeneratorName = MethodBase.GetCurrentMethod().Name;
genE.GeneratorName = MethodBase.GetCurrentMethod().Name;
genG.GeneratorName = MethodBase.GetCurrentMethod().Name;
}
//Setup helpers
private Dictionary <Type, EntityContext> GetEntityContexts(
Dictionary <Type, long> entityCurrentCount, long targetCount)
{
var generatorSetup = new GeneratorSetup();
var category = new EntityDescription <Category>()
.SetTargetCount(targetCount);
var post = new EntityDescription <Post>()
.SetTargetCount(targetCount)
.SetRequired(typeof(Category));
var comment = new EntityDescription <Comment>()
.SetTargetCount(targetCount)
.SetRequired(typeof(Post))
.SetRequired(typeof(Category));
Dictionary <Type, IEntityDescription> dictDescriptions = new List <IEntityDescription>
{
category,
post,
comment
}.ToDictionary(x => x.Type, x => x);
Dictionary <Type, EntityContext> contexts =
generatorSetup.SetupEntityContexts(dictDescriptions);
contexts.Values.ToList().ForEach(x =>
{
x.EntityProgress.AddFlushedCount(0);
x.EntityProgress.NextFlushCount = 0;
});
foreach (KeyValuePair <Type, long> entity in entityCurrentCount)
{
contexts[entity.Key].EntityProgress.CurrentCount = entity.Value;
}
return(contexts);
}
//init
public GenerateStorageIdsCommand(EntityContext entityContext, GeneratorSetup setup)
{
_entityContext = entityContext;
_setup = setup;
}
//init
public FinishCommand(GeneratorSetup setup,
Dictionary <Type, EntityContext> entityContexts)
{
_setup = setup;
_entityContexts = entityContexts;
}
//init
public Validator(GeneratorSetup generatorSetup)
{
_generatorSetup = generatorSetup;
}
private static void GenerateRatioCapacityToWeightSum(ICollection<GeneratorSetup> list)
{
for (var f = 0.1f; f <= 1.0f; f += 0.05f)
{
GeneratorSetup gen = new GeneratorSetup();
gen.GeneratorName = MethodBase.GetCurrentMethod().Name;
gen.RatioCapacityToWeightSum = f;
list.Add(gen);
}
}
//progress
private void PrintProgress(GeneratorSetup setup, decimal percent)
{
Debug.WriteLine($"Data generation progress {percent.ToString("F")}%");
}
private static void PrintProgress(GeneratorSetup setup, decimal percent)
{
Console.CursorTop = 0;
Console.WriteLine(percent.ToString("F"));
}