Beispiel #1
0
        public static async Task UploadDocuments(List <Document> documents, string activeEtag, Action <int, int> progress)
        {
            if (documents == null || !documents.Any())
            {
                return;
            }

            // upload documents
            await ParallelUtility.ParallelForEach(documents, document =>
            {
                return(CosmosDBAccessor <Document> .UpsertAsync(document));
            }, 400, 200, progress);

            // switch active etag
            var doc    = documents.First();
            var active = new Active
            {
                ActiveEtag = activeEtag,
                IsActive   = false,
                Branch     = doc.Branch,
                Locale     = doc.Locale,
                Docset     = doc.Docset,
                id         = HashUtility.GetSha1HashString($"{doc.Docset}|{doc.Branch}|{doc.Locale}")
            };

            await CosmosDBAccessor <Active> .UpsertAsync(active);
        }
Beispiel #2
0
        public static async Task Migrate(string basePath, string branch, string locale, int top = 3)
        {
            Console.WriteLine($"Get depots for {basePath}");
            var depots = await s_dhsClient.GetAllDepotsBySiteBasePath("Docs", basePath, null, CancellationToken.None);

            var topDepots  = depots.OrderBy(d => d.Priority).Take(top);
            var activeEtag = Guid.NewGuid().ToString();

            await ParallelUtility.ParallelForEach(topDepots, async topDepot =>
            {
                var continueAt = string.Empty;
                var documents  = new List <GetDocumentResponse>();
                var i          = 1;
                do
                {
                    Console.WriteLine($"Load {1000 * i++} documents for {topDepot.DepotName}");
                    var documentsResponse = await s_dhsClient.GetDocumentsPaginated(topDepot.DepotName, locale, branch, false, continueAt, null, 1000, CancellationToken.None);
                    documents.AddRange(documentsResponse.Documents);
                    continueAt = documentsResponse.ContinueAt;
                } while (!string.IsNullOrEmpty(continueAt));

                Console.WriteLine($"Convert {documents.Count} documents for {topDepot.DepotName}");
                var pageDocs = new ConcurrentBag <Document>();
                await ParallelUtility.ParallelForEach(documents, async document =>
                {
                    using (Stream contentStream = await HttpHelper.DownloadAsStream(document.ContentUri, CancellationToken.None))
                    {
                        var(pageUrl, pageHash) = await Writer.UploadPage(contentStream, document.CombinedMetadata.GetValueOrDefault <bool>("is_dynamic_rendering"), document.CombinedMetadata.GetValueOrDefault <string>("content_type"));
                        var pageDoc            = new Document
                        {
                            Docset             = topDepot.DepotName,
                            Url                = $"{basePath}{document.AssetId}",
                            Locale             = locale,
                            Branch             = branch,
                            Monikers           = document.CombinedMetadata.GetValueOrDefault <JArray>("monikers")?.ToObject <List <string> >(),
                            ActiveEtag         = activeEtag,
                            PageHash           = pageHash,
                            PageUrl            = pageUrl,
                            PageType           = document.CombinedMetadata.GetValueOrDefault <string>("page_type"),
                            Title              = document.CombinedMetadata.GetValueOrDefault <string>("title"),
                            Layout             = document.CombinedMetadata.GetValueOrDefault <string>("layout"),
                            IsDynamicRendering = document.CombinedMetadata.GetValueOrDefault <bool>("is_dynamic_rendering"),
                            ContentType        = document.CombinedMetadata.GetValueOrDefault <string>("content_type")
                        };
                        pageDocs.Add(pageDoc);
                    }
                },
                                                      400,
                                                      200,
                                                      (done, total) =>
                {
                    var percent = ((int)(100 * Math.Min(1.0, done / Math.Max(1.0, total)))).ToString();
                    Console.WriteLine($"Uploading Page Content for {topDepot.DepotName}: {percent.PadLeft(3)}% {done}/{total}");
                });

                await Writer.UploadDocuments(pageDocs.ToList(), activeEtag, (done, total) =>
                {
                    var percent = ((int)(100 * Math.Min(1.0, done / Math.Max(1.0, total)))).ToString();
                    Console.WriteLine($"Uploading Page Document for {topDepot.DepotName}: {percent.PadLeft(3)}% {done}/{total}");
                });
            }, 10, 5);
        }