public ProcessorSettings GetSettings()
{
var telegramSetting = new TelegramSettings {
Enabled = GetBool("TelegramEnabled"),
Token = GetString("TelegramBotToken"),
ChatId = GetString("TelegramChatId")
};
var elasticSearchClientSettings = new ElasticSearchClientSettings {
ComponentForIndex = ComponentType.ElasticSearch,
ElasticSearchUrl = GetString("ElasticSearchClientUrl"),
DefaultIndex = GetString("ElasticSearchClientIndex"),
FrameSize = GetInt("ElasticSearchClientFrameSize"),
ShopName = GetString("ElasticSearchClientShopName")
};
var settings = new ProcessorSettings {
AttemptsToDownload = GetInt("AttemptsToDownload"),
EnableExtendedStatistics = GetBool("EnableExtendedStatistics"),
DirectoryPath = GetString("DirectoryPath"),
DuplicateFile = GetString("DuplicateFile"),
ShowStatistics = GetBool("ShowStatistics"),
ElasticSearchClientSettings = elasticSearchClientSettings,
TelegramSettings = telegramSetting
};
return(settings);
}
public ProductsHandler(
ElasticSearchClientSettings settings,
BackgroundBaseContext context,
BackgroundWorks works,
DbHelper db)
: base(settings, works, db)
{
_context = context;
}
public ProductLinker(
ElasticSearchClientSettings settings,
DbHelper dbHelper,
BackgroundBaseContext context)
: base(ComponentType.ProductLinker, context)
{
_elasticClient = new ElasticSearchClient <Product>(settings, context);
_dbHelper = dbHelper;
}
static void Main(string[] args)
{
var elasticSettings = new ElasticSearchClientSettings {
ComponentForIndex = ComponentType.ElasticSearch,
DefaultIndex = "products-old-1",
ElasticSearchUrl = "http://185.221.152.127:9200",
FrameSize = 20000,
ShopName = "OldTheStore"
};
var swCash = new Stopwatch();
swCash.Start();
var db1 = new ImportDbContext();
var cashTask = Task.Run(Cash.FillCash);
var offerTask = Task.Run(
() => db1.Offers.Where(o => o.Enabled == 1).Select(
o => new OfferId {
Id = o.Id,
Photo1 = o.Photo1
}).ToList());
Task.WaitAll(cashTask, offerTask);
//Task.WaitAll( offerTask );
var allIds = offerTask.Result;
//Cash.FillCash();
// var offersByPhoto = db.Offers.Where( o => o.Enabled == 1 ).Select(
// o => new OfferId {
// Id = o.Id,
// Photo1 = o.Photo1
// } ).ToList();
var groups = allIds.GroupBy(oi => oi.Photo1).Select(g => g.ToList()).ToList();
swCash.Stop();
var timeCash = swCash.ElapsedMilliseconds;
var skip = 0;
var step = 15000;
while (skip <= groups.Count)
{
var swIteration = new Stopwatch();
swIteration.Start();
var ids = groups.Skip(skip).Take(step)
.SelectMany(g => g.Select(o => o.Id)).ToHashSet();
//var part = db.Offers.Where( o => ids.Contains( o.Id ) ).ToList();
var db2 = new ImportDbContext();
var part = new List <OfferOld>();
var querySkip = 0;
var queryStep = 50000;
while (querySkip < ids.Count)
{
var idsPart = ids.Skip(querySkip).Take(queryStep).ToList();
part.AddRange(db2.Offers.Where(o => idsPart.Contains(o.Id)));
querySkip += queryStep;
}
var partCash = Cash.Values.Where(p => ids.Contains(p.Key))
.ToDictionary(p => p.Key, p => p.Value);
var cleanOffers = part.Where(o => string.IsNullOrWhiteSpace(o.Photo1) == false).ToList();
if (cleanOffers.Any() == false)
{
skip += step;
continue;
}
var products = Converter.GetProducts(cleanOffers, partCash);
var elasticClient = new ElasticSearchClient <Product>(
elasticSettings,
new BackgroundBaseContext("id", "name"));
elasticClient.DoBulkAllForImport(products);
skip += step;
swIteration.Stop();
var time = swIteration.ElapsedMilliseconds;
}
}
public TagsWorker(ElasticSearchClientSettings settings, BackgroundWorks works, DbHelper dbHelper)
: base(settings, works, dbHelper)
{
}
private static IElasticClient <IIndexedEntities> CreateElasticClient(BackgroundBaseContext context, ElasticSearchClientSettings settings) => new ElasticSearchClient <IIndexedEntities>(settings, context);
public static async Task <IActionResult> GetTotalStatistics(DateTime?startPoint, ElasticSearchClientSettings setting)
{
var client = CreateElasticClient(new BackgroundBaseContext("1", "none"), setting);
var statistics = new TotalStatistics {
Products = await Task.Run(() => client.CountProducts()),
SoldOut = await Task.Run(() => client.CountSoldOutProducts()),
CountForSoldOut = startPoint != null
? await Task.Run(() => client.CountForDisableProducts(startPoint.Value, null))
: 0
};
return(new JsonResult(statistics));
}
public ProductWorker(ElasticSearchClientSettings settings)
{
_settings = settings;
}
protected BaseLinkWorker(ElasticSearchClientSettings settings, BackgroundWorks works, DbHelper db)
{
_settings = settings;
Works = works;
Db = db;
}
public CategoryWorker(ElasticSearchClientSettings settings, BackgroundWorks works, DbHelper db)
: base(settings, works, db)
{
}
