Esempio n. 1
0
        static void Main(string[] args)
        {
            var elasticSettings = new ElasticSearchClientSettings {
                ComponentForIndex = ComponentType.ElasticSearch,
                DefaultIndex      = "products-old-1",
                ElasticSearchUrl  = "http://185.221.152.127:9200",
                FrameSize         = 20000,
                ShopName          = "OldTheStore"
            };

            var swCash = new Stopwatch();

            swCash.Start();

            var db1 = new ImportDbContext();

            var cashTask  = Task.Run(Cash.FillCash);
            var offerTask = Task.Run(
                () => db1.Offers.Where(o => o.Enabled == 1).Select(
                    o => new OfferId {
                Id     = o.Id,
                Photo1 = o.Photo1
            }).ToList());

            Task.WaitAll(cashTask, offerTask);
            //Task.WaitAll(  offerTask );
            var allIds = offerTask.Result;

            //Cash.FillCash();

            // var offersByPhoto = db.Offers.Where( o => o.Enabled == 1 ).Select(
            //     o => new OfferId {
            //         Id = o.Id,
            //         Photo1 = o.Photo1
            //     } ).ToList();

            var groups = allIds.GroupBy(oi => oi.Photo1).Select(g => g.ToList()).ToList();

            swCash.Stop();
            var timeCash = swCash.ElapsedMilliseconds;

            var skip = 0;
            var step = 15000;

            while (skip <= groups.Count)
            {
                var swIteration = new Stopwatch();
                swIteration.Start();

                var ids = groups.Skip(skip).Take(step)
                          .SelectMany(g => g.Select(o => o.Id)).ToHashSet();

                //var part = db.Offers.Where( o => ids.Contains( o.Id ) ).ToList();

                var db2 = new ImportDbContext();

                var part      = new List <OfferOld>();
                var querySkip = 0;
                var queryStep = 50000;
                while (querySkip < ids.Count)
                {
                    var idsPart = ids.Skip(querySkip).Take(queryStep).ToList();
                    part.AddRange(db2.Offers.Where(o => idsPart.Contains(o.Id)));

                    querySkip += queryStep;
                }

                var partCash = Cash.Values.Where(p => ids.Contains(p.Key))
                               .ToDictionary(p => p.Key, p => p.Value);

                var cleanOffers = part.Where(o => string.IsNullOrWhiteSpace(o.Photo1) == false).ToList();
                if (cleanOffers.Any() == false)
                {
                    skip += step;
                    continue;
                }
                var products = Converter.GetProducts(cleanOffers, partCash);

                var elasticClient = new ElasticSearchClient <Product>(
                    elasticSettings,
                    new BackgroundBaseContext("id", "name"));

                elasticClient.DoBulkAllForImport(products);
                skip += step;

                swIteration.Stop();
                var time = swIteration.ElapsedMilliseconds;
            }
        }