private void GenerateProperties(UmbracoDbContext context, int[] definitionIds,
IEnumerable <UCommerceProductCatalogGroup> stores)
{
Console.Write($"Generating properties for {Count:N0} stores. ");
using (var p = new ProgressBar())
{
var mediaIds = _cmsContent.GetAllMediaIds(context);
var contentIds = _cmsContent.GetAllContentIds(context);
var languageCodes = _cmsContent.GetLanguageIsoCodes(context);
var definitionFields = LookupDefinitionFields(context, definitionIds);
uint batchSize = 100_000;
uint numberOfBatches = definitionFields.Any()
? (uint)Math.Ceiling(1.0 * stores.Count() * (uint)definitionFields.Average(x => x.Count()) /
batchSize)
: 1;
var propertyBatches = stores
.Where(store => store.DefinitionId.HasValue)
.SelectMany(store => definitionFields[store.DefinitionId.Value].SelectMany(field =>
AddEntityProperty(store.Guid, field.Field, languageCodes, mediaIds,
contentIds, field.Editor, field.Enums)))
.Batch(batchSize);
propertyBatches.EachWithIndex((properties, index) =>
{
context.BulkInsert(properties.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / numberOfBatches);
});
}
}
private void GenerateAllowedPriceGroups(UmbracoDbContext context, IEnumerable <UCommerceProductCatalog> catalogs,
int[] priceGroupIds)
{
Console.Write($"Generating allowed price groups for {Count:N0} catalogs. ");
using (var p = new ProgressBar())
{
uint batchSize = 100_000;
uint numberOfBatches =
(uint)Math.Ceiling(1.0 * catalogs.Count() * (uint)priceGroupIds.Length / batchSize);
var allowedPriceGroupBatches = catalogs.SelectMany(catalog =>
priceGroupIds.Select(priceGroupId =>
_faker.Random.Bool()
? new UCommerceProductCatalogPriceGroupRelation
{
PriceGroupId = priceGroupId, ProductCatalogId = catalog.ProductCatalogId,
Guid = _faker.Random.Guid()
}
: null
)
.Compact()).Batch(batchSize);
allowedPriceGroupBatches.EachWithIndex((allowedPriceGroups, index) =>
{
context.BulkInsert(allowedPriceGroups.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / numberOfBatches);
});
}
}
public override void Seed(UmbracoDbContext context)
{
int[] productIds = context.UCommerceProduct
.Where(product => product.UCommerceCategoryProductRelation.Count == 0)
.Where(product => product.ParentProductId == null) // not variants
.Select(x => x.ProductId).ToArray();
int[] categoryIds = context.UCommerceCategory.Select(x => x.CategoryId).ToArray();
uint batchSize = 100_000;
uint batchCount = (uint)Math.Ceiling(1.0 * Count / batchSize);
Console.Write(
$"Generating {Count:N0} relations for {productIds.Length:N0} products and {categoryIds.Length:N0} categories in batches of {batchSize:N0}. ");
using (var p = new ProgressBar())
{
var relationBatches = GeneratorHelper
.Generate(() => GenerateRelation(productIds, categoryIds), Count)
.DistinctBy(a => a.UniqueIndex())
.Batch(batchSize);
relationBatches.EachWithIndex((relations, index) =>
{
context.BulkInsert(relations.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / batchCount);
});
}
}
private void GenerateDescriptions(UmbracoDbContext context, IEnumerable <UCommerceCategory> categories,
string[] languageCodes)
{
uint batchSize = 100_000;
uint numberOfBatches =
(uint)Math.Ceiling(1.0 * categories.Count() * (uint)languageCodes.Length / batchSize);
Console.Write(
$"Generating {categories.Count() * languageCodes.Length:N0} descriptions for {categories.Count():N0} categories in batches of {batchSize:N0}. ");
using (var p = new ProgressBar())
{
var descriptionBatches = categories.SelectMany(category =>
languageCodes.Select(language => _descriptionFaker
.RuleFor(x => x.CultureCode, f => language)
.RuleFor(x => x.CategoryId, f => category.CategoryId)
.RuleFor(x => x.DisplayName, f => $"{category.Name} {f.Commerce.Product()}")
.Generate()
)).Batch(batchSize);
descriptionBatches.EachWithIndex((descriptions, index) =>
{
context.BulkInsert(descriptions.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / numberOfBatches);
});
}
}
private void GenerateProperties(UmbracoDbContext context, int[] definitionIds,
IEnumerable <UCommerceProductCatalog> catalogs,
string[] languageCodes)
{
Console.Write($"Generating properties for {Count:N0} catalogs...");
using (var p = new ProgressBar())
{
var mediaIds = _cmsContent.GetAllMediaIds(context);
var contentIds = _cmsContent.GetAllContentIds(context);
var definitionFields = LookupDefinitionFields(context, definitionIds);
uint batchSize = 100_000;
uint numberOfBatches = definitionFields.Any()
? (uint)Math.Ceiling(
1.0 * batchSize / definitionFields.Average(x => x.Count()) / catalogs.Count())
: 1;
var propertiyBatches = catalogs
.Where(catalog => catalog.DefinitionId.HasValue)
// ReSharper disable once PossibleInvalidOperationException
.SelectMany(category => definitionFields[category.DefinitionId.Value].SelectMany(field =>
AddEntityProperty(category.Guid, field.Field, languageCodes, mediaIds,
contentIds, field.Editor, field.Enums)))
.Batch(batchSize);
propertiyBatches.EachWithIndex((properties, index) =>
{
context.BulkInsert(properties.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / numberOfBatches);
});
}
}
private void GenerateProperties(UmbracoDbContext context, int[] definitionIds,
IEnumerable <UCommerceCategory> categories,
string[] languageCodes, string[] mediaIds)
{
var definitionFields = LookupDefinitionFields(context, definitionIds);
uint estimatedPropertyCount = definitionFields.Any()
? (uint)definitionFields.Average(x => x.Count()) * (uint)categories.Count()
: 1;
uint batchSize = 1_000_000;
uint numberOfBatches = (uint)Math.Ceiling(1.0 * estimatedPropertyCount / batchSize);
Console.Write(
$"Generating ~{estimatedPropertyCount:N0} properties for {categories.Count():N0} categories. ");
using (var p = new ProgressBar())
{
var contentIds = _cmsContent.GetAllContentIds(context);
var propertyBatches = categories.SelectMany(category =>
definitionFields[category.DefinitionId].SelectMany(field =>
AddCategoryProperty(category.CategoryId, field.Field, languageCodes,
mediaIds, contentIds, field.Editor, field.Enums)))
.Batch(batchSize);
propertyBatches.EachWithIndex((properties, index) =>
{
context.BulkInsert(properties.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / numberOfBatches);
});
}
}
private List <UCommerceCategory> GenerateSubCategories(UmbracoDbContext context,
int[] definitionIds, string[] mediaIds, IEnumerable <UCommerceCategory> topLevelCategories)
{
uint batchSize = 100_000;
uint numberOfBatches = (uint)Math.Ceiling(4.0 * Count / 5.0 / batchSize);
Console.Write($"Generating {4 * Count / 5:N0} subcategories in batches of {batchSize}. ");
var insertedCategories = new List <UCommerceCategory>((int)Count / 5);
using (var p = new ProgressBar())
{
var categoryBatches = GeneratorHelper
.Generate(() => GenerateSubCategory(definitionIds, mediaIds, topLevelCategories), 4 * Count / 5)
.DistinctBy(a => a.UniqueIndex())
.Batch(batchSize);
categoryBatches.EachWithIndex((categories, index) =>
{
var listOfCats = categories.ToList();
context.BulkInsert(listOfCats, options => options.SetOutputIdentity = true);
insertedCategories.AddRange(listOfCats);
p.Report(1.0 * index / numberOfBatches);
});
return(insertedCategories);
}
}
public override void Seed(UmbracoDbContext context)
{
var languageCodes = _cmsContent.GetLanguageIsoCodes(context);
var productDefinitionFields = LookupProductDefinitionFields(context, true);
var priceGroupIds = context.UCommercePriceGroup.Select(pg => pg.PriceGroupId).ToArray();
var productFamilyIds = context.UCommerceProduct
.Where(p => p.ProductDefinition.UCommerceProductDefinitionField.Any(f =>
f.IsVariantProperty)) // pick families only
.Where(p => p.ParentProductId == null) // don't pick variants
.Select(product => new ProductWithDefinition
{
ProductId = product.ProductId, ProductDefinitionId = product.ProductDefinitionId, Sku = product.Sku
})
.ToArray();
var mediaIds = _cmsContent.GetAllMediaIds(context);
var contentIds = _cmsContent.GetAllMediaIds(context);
var products = GenerateVariants(context, productFamilyIds, mediaIds);
GenerateDescriptions(context, languageCodes, products);
GenerateProperties(context, products, productDefinitionFields, mediaIds, contentIds);
GeneratePrices(context, priceGroupIds, products);
}
private void GenerateRelations(UmbracoDbContext context, IEnumerable <UCommerceProduct> products,
int[] productRelationTypeIds)
{
Console.Write(
$"Generating {_databaseSize.ProductRelationsPerProduct * Count:N0} relations for {Count:N0} {EntityNamePlural}. ");
using (var p = new ProgressBar())
{
uint batchSize = 100_000;
uint numberOfBatches = 1 + _databaseSize.ProductRelationsPerProduct * Count / batchSize;
var relationBatches = products.SelectMany(product => Enumerable.Range(1,
(int)_databaseSize.ProductRelationsPerProduct).Select(i =>
{
var otherProduct = _faker.PickRandom(products);
if (otherProduct != product)
{
return(GenerateProductRelation(product.ProductId, otherProduct.ProductId,
productRelationTypeIds));
}
return(null);
}))
.Compact()
.DistinctBy(a => a.UniqueIndex())
.Batch(batchSize);
relationBatches.EachWithIndex((relations, index) =>
{
context.BulkInsert(relations.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / numberOfBatches);
});
}
}
protected IList <UCommerceProduct> GenerateVariants(UmbracoDbContext context, ProductWithDefinition[] products,
string[] mediaIds)
{
uint batchSize = 100_000;
uint numberOfBatches = (uint)Math.Ceiling(1.0 * Count / batchSize);
Console.Write($"Generating {Count:N0} {EntityNamePlural} in {numberOfBatches} batches of {batchSize}. ");
var insertedProducts = new List <UCommerceProduct>((int)Count);
using (var p = new ProgressBar())
{
var variantBatches =
GeneratorHelper.Generate(() => GenerateVariant(mediaIds, products),
Count)
.DistinctBy(a => a.UniqueIndex())
.Batch(batchSize);
variantBatches.EachWithIndex((variants, index) =>
{
var listOfVariants = variants.ToList();
context.BulkInsert(listOfVariants, options => options.SetOutputIdentity = true);
insertedProducts.AddRange(listOfVariants);
p.Report(1.0 * index / numberOfBatches);
});
return(insertedProducts);
}
}
public override void Seed(UmbracoDbContext context)
{
var definitionIds = context.UCommerceDefinition.Select(x => x.DefinitionId).ToArray();
var dataTypeIds = context.UCommerceDataType.Select(x => x.DataTypeId).ToArray();
var fields = GenerateFields(context, definitionIds, dataTypeIds);
GenerateDescriptions(context, fields);
}
protected static ILookup <int, ProductDefinitionFieldEditorAndEnum> LookupProductDefinitionFields(
UmbracoDbContext context, bool variantProperties)
{
return(context.UCommerceProductDefinitionField
.Where(field => field.IsVariantProperty == variantProperties)
.Select(field => new ProductDefinitionFieldEditorAndEnum(field, field.DataType.DefinitionName,
field.DataType.UCommerceDataTypeEnum.Select(x => x.Guid)))
.ToLookup(field => field.Field.ProductDefinitionId));
}
private List <UCommerceCurrency> GetDefaultCurrenciesNotInDb(UmbracoDbContext context)
{
var preSeededCurrencies = context.UCommerceCurrency.ToList();
return(DefaultCurrencies.Where(x => !preSeededCurrencies
.Select(y => y.Isocode)
.Contains(x.Isocode))
.ToList());
}
protected void GeneratePrices(UmbracoDbContext context, int[] priceGroupIds,
IEnumerable <UCommerceProduct> products)
{
ulong totalCount = Count * (ulong)priceGroupIds.Length * _databaseSize.TiersPerPriceGroup;
uint batchSize = 1_00_000;
Console.Write(
$"Generating {totalCount:N0} prices in batches of {batchSize:N0}. ");
using (var p = new ProgressBar())
{
var priceBatches = products
.SelectMany(product =>
priceGroupIds.SelectMany(priceGroupId =>
Enumerable.Range(1, (int)_databaseSize.TiersPerPriceGroup)
.Select(i => GeneratePrice(priceGroupId))
)
).Batch(batchSize);
var productBatches = products
.SelectMany(product =>
priceGroupIds.SelectMany(priceGroupId =>
Enumerable.Range(1, (int)_databaseSize.TiersPerPriceGroup)
.Select(i => new
{ Product = product, Tier = i })
)
).Batch(batchSize);
ulong numBatches = 2 * totalCount / batchSize + 1;
uint batchCount = 0;
foreach (var batch in priceBatches.Zip(productBatches, (prices, moreInfo) => new { prices, moreInfo }))
{
var prices = batch.prices.ToList();
context.BulkInsert(prices,
options => options.SetOutputIdentity = true);
p.Report(1.0 * ++batchCount / numBatches);
var productPrices = prices.Zip(batch.moreInfo,
(price, product) => new { price, product }).Select(zippedBatch =>
GenerateProductPrice(zippedBatch.price.PriceId, zippedBatch.product.Product.ProductId,
zippedBatch.product.Tier)).ToArray();
context.BulkInsert(productPrices,
options =>
{
options.SetOutputIdentity = false;
// Specifying the columns to insert is necessary for this table only.
// The reason is unknown, but if you omit it, MinimumQuantity will always be '1'.
options.PropertiesToInclude =
new [] { "MinimumQuantity", "Guid", "ProductId", "PriceId" }.ToList();
});
p.Report(1.0 * ++batchCount / numBatches);
}
}
}
public override void Seed(UmbracoDbContext context)
{
var definitionIds = context.UCommerceDefinition
.Where(d => d.DefinitionTypeId == (int)DefinitionType.CatalogGroup).Select(c => c.DefinitionId)
.ToArray();
var stores = GenerateStores(context, definitionIds);
GenerateProperties(context, definitionIds, stores);
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count:N0} product relation types. ");
using (var p = new ProgressBar())
{
var dataTypes = GeneratorHelper.Generate(Generate, Count);
p.Report(0.5);
context.BulkInsert(dataTypes, options => options.SetOutputIdentity = false);
}
}
private List <UCommerceDataType> GenerateDataTypes(UmbracoDbContext context, int[] definitionIds)
{
Console.Write($"Generating {Count:N0} data types.");
using (var p = new ProgressBar())
{
var dataTypes = GeneratorHelper.Generate(() => Generate(definitionIds), Count).ToList();
p.Report(0.5);
context.BulkInsert(dataTypes);
return(dataTypes);
}
}
public override void Seed(UmbracoDbContext context)
{
var definitionIds = context.UCommerceDefinition
.Where(d => d.DefinitionTypeId == (int)DefinitionType.DataType).Select(c => c.DefinitionId)
.ToArray();
var definitionFields = LookupDefinitionFields(context, definitionIds);
var dataTypes = GenerateDataTypes(context, definitionIds);
GenerateProperties(context, dataTypes, definitionFields);
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count:N0} order number series. ");
using (var p = new ProgressBar())
{
p.Report(0.1);
var orderNumberSeries =
GeneratorHelper.Generate(() => _faker.Generate(), Count);
p.Report(0.5);
context.BulkInsert(orderNumberSeries);
}
}
public int Run()
{
var seeder = new DataSeeder(_dbSize, _excludeCmsTables);
seeder.Seed(() =>
{
var dbContext = new UmbracoDbContext(_connectionString, _verbose);
dbContext.ChangeTracker.AutoDetectChangesEnabled = false;
return(dbContext);
});
return(0);
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count:N0} product definitions. ");
using (var p = new ProgressBar())
{
var productDefinitions = GeneratorHelper.Generate(Generate, Count);
p.Report(0.33);
productDefinitions.ConsecutiveSortOrder((def, val) => { def.SortOrder = (int) val; });
p.Report(0.66);
context.BulkInsert(productDefinitions, options => options.SetOutputIdentity = false);
}
}
private List <UCommerceDefinitionField> GenerateFields(UmbracoDbContext context, int[] definitionIds,
int[] dataTypeIds)
{
Console.Write($"Generating {Count:N0} definition fields. ");
using (var p = new ProgressBar())
{
var fields = GeneratorHelper.Generate(() => GenerateField(definitionIds, dataTypeIds), Count).ToList();
fields.ConsecutiveSortOrder((f, v) => { f.SortOrder = (int)v; });
p.Report(0.5);
context.BulkInsert(fields);
return(fields);
}
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count:N0} definitions. ");
using (var p = new ProgressBar())
{
var catalogDefinitions = Enum.GetValues(typeof(DefinitionType))
.Cast <DefinitionType>()
.SelectMany(definitionType => GeneratorHelper.Generate(() => Generate(definitionType), Count));
p.Report(0.33);
catalogDefinitions.ConsecutiveSortOrder((f, v) => { f.SortOrder = (int)v; });
p.Report(0.66);
context.BulkInsert(catalogDefinitions.ToList(), options => options.SetOutputIdentity = false);
}
}
public override void Seed(UmbracoDbContext context)
{
var definitionIds = context.UCommerceDefinition
.Where(d => d.DefinitionTypeId == (int)DefinitionType.Catalog).Select(c => c.DefinitionId)
.ToArray();
var languageCodes = _cmsContent.GetLanguageIsoCodes(context);
var priceGroupIds = context.UCommercePriceGroup.Select(x => x.PriceGroupId).ToArray();
var catalogs = GenerateCatalogs(context, definitionIds, priceGroupIds);
GenerateDescriptions(context, catalogs, languageCodes);
GenerateProperties(context, definitionIds, catalogs, languageCodes);
GenerateAllowedPriceGroups(context, catalogs, priceGroupIds);
}
protected void GenerateProperties(UmbracoDbContext context, IEnumerable <UCommerceProduct> products,
ILookup <int, ProductDefinitionFieldEditorAndEnum> productDefinitionFields, string[] mediaIds,
string[] contentIds)
{
uint averageNumberOfFieldsPerProduct = productDefinitionFields.Any()
? (uint)productDefinitionFields.Average(f => f.Count()) / 2
: 1;
uint batchSize = 1_000_000;
uint estimatedBatchCount = (uint)Math.Ceiling(1.0 * Count * averageNumberOfFieldsPerProduct / batchSize);
Console.Write(
$"Generating ~{averageNumberOfFieldsPerProduct * products.Count():N0} language variant properties with values for {products.Count():N0} {EntityNamePlural}. ");
using (var p = new ProgressBar())
{
ILookup <int, int> descriptions = context.UCommerceProductDescription
.Select(x => new { x.ProductId, x.ProductDescriptionId })
.ToLookup(x => x.ProductId, x => x.ProductDescriptionId);
var languageVariantPropertyBatches = products.SelectMany(product =>
GenerateLanguageVariantProductProperties(descriptions[product.ProductId],
productDefinitionFields[product.ProductDefinitionId], mediaIds, contentIds))
.Batch(batchSize);
languageVariantPropertyBatches.EachWithIndex((languageVariantProperties, index) =>
{
context.BulkInsert(languageVariantProperties.ToList(),
options => options.SetOutputIdentity = false);
p.Report(1.0 * index / estimatedBatchCount);
});
}
Console.Write(
$"Generating ~{averageNumberOfFieldsPerProduct * products.Count():N0} language invariant properties with values for {products.Count():N0} {EntityNamePlural}. ");
using (var p = new ProgressBar())
{
var simplePropertyBatches = products.SelectMany(product =>
GenerateLanguageInvariantProductProperties(product.ProductId,
productDefinitionFields[product.ProductDefinitionId], mediaIds, contentIds))
.Batch(batchSize);
simplePropertyBatches.EachWithIndex((simpleProperties, index) =>
{
context.BulkInsert(simpleProperties.ToList(), options => options.SetOutputIdentity = false);
p.Report(1.0 * index / estimatedBatchCount);
});
}
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count} languages. ");
using (var p = new ProgressBar())
{
var availableCultures = CultureInfo.GetCultures(CultureTypes.AllCultures)
.Where(cultureInfo => !context.UmbracoLanguage
.Any(language => language.LanguageIsocode == cultureInfo.Name))
.ToArray();
_pickedCultures = _faker.PickRandom(availableCultures, (int)Count).ToArray();
var umbracoLanguages = GeneratorHelper.Generate(Generate, Count).Where(x => x != null).ToList();
p.Report(0.5);
context.BulkInsert(umbracoLanguages, options => options.SetOutputIdentity = false);
}
}
private List <UCommerceProductCatalog> GenerateCatalogs(UmbracoDbContext context, int[] definitionIds,
int[] priceGroupIds)
{
Console.Write($"Generating {Count:N0} catalogs. ");
using (var p = new ProgressBar())
{
var storeIds = context.UCommerceProductCatalogGroup.Select(c => c.ProductCatalogGroupId).ToArray();
var catalogs =
GeneratorHelper.Generate(() => GenerateCatalog(definitionIds, storeIds, priceGroupIds), Count)
.DistinctBy(a => a.UniqueIndex())
.ToList();
p.Report(0.5);
context.BulkInsert(catalogs, options => options.SetOutputIdentity = true);
return(catalogs);
}
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count} currencies. ");
using (var p = new ProgressBar())
{
var defaultCurrenciesNotInDb = GetDefaultCurrenciesNotInDb(context);
var currenciesLeft = Math.Max(0, Count - defaultCurrenciesNotInDb.Count);
var currencies = currenciesLeft == 0
? defaultCurrenciesNotInDb.Take((int)Count)
: defaultCurrenciesNotInDb.Concat(GeneratorHelper.Generate(GenerateCurrency, (uint)currenciesLeft))
.DistinctBy(x => x.Isocode);
p.Report(0.5);
context.BulkInsert(currencies.ToList(), options => options.SetOutputIdentity = false);
}
}
private void GenerateDescriptions(UmbracoDbContext context, IEnumerable <UCommerceDefinitionField> fields)
{
Console.Write($"Generating descriptions for {Count:N0} definition fields. ");
using (var p = new ProgressBar())
{
var languageCodes = _cmsContent.GetLanguageIsoCodes(context);
var descriptions = fields.SelectMany(field =>
languageCodes.Select(language =>
_definitionFieldDescriptionFaker
.RuleFor(x => x.DefinitionFieldId, f => field.DefinitionFieldId)
.RuleFor(x => x.CultureCode, f => language)
.Generate()
)
);
p.Report(0.5);
context.BulkInsert(descriptions.ToList(), options => options.SetOutputIdentity = false);
}
}
public override void Seed(UmbracoDbContext context)
{
Console.Write($"Generating {Count:N0} price groups. ");
using (var p = new ProgressBar())
{
var currencyIds = context.UCommerceCurrency.Select(x => new Tuple <int, string>(x.CurrencyId, x.Isocode))
.ToArray();
p.Report(0.1);
var priceGroups = GeneratorHelper.Generate(() => Generate(currencyIds), Count)
.DistinctBy(x => x.UniqueIndex())
.ToArray();
p.Report(0.5);
context.BulkInsert(priceGroups, options => options.SetOutputIdentity = false);
}
}
