Example #1
0
        /// <summary>
        /// from http://cudl.lib.cam.ac.uk
        /// </summary>
        /// <param name="resourceNumber">MS-RAS-00258</param>
        /// <param name="friendlyUrl"></param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromCambridge(string resourceNumber, string friendlyUrl)
        {
            string url = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}.json";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.Cambridge && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated artifact friendly url '{friendlyUrl}'"));
            }



            ImportJob job = new ImportJob()
            {
                JobType         = JobType.Cambridge,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();


            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(Configuration))
                                    {
                                        if (
                                            (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync())
                                            !=
                                            null
                                            )
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "aborted because of duplicated friendly url";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string json = await result.Content.ReadAsStringAsync();
                                        var parsed  = JObject.Parse(json);
                                        book.Name   = book.NameInEnglish =
                                            parsed.SelectToken("logicalStructures[*].label").Value <string>();

                                        book.Description = book.DescriptionInEnglish =
                                            Regex.Replace(
                                                parsed.SelectToken("descriptiveMetadata[*].abstract.displayForm").Value <string>(),
                                                "<.*?>", string.Empty);

                                        int tagOrder = 1;
                                        foreach (JToken descriptiveMetadata in parsed.SelectTokens("$.descriptiveMetadata[*]").Children())
                                        {
                                            foreach (JToken child in descriptiveMetadata.Children())
                                            {
                                                if (child.SelectToken("label") != null && child.SelectToken("display") != null)
                                                {
                                                    if (child.SelectToken("display").Value <string>() == "True")
                                                    {
                                                        string metaName  = child.SelectToken("label").Value <string>();
                                                        string metaValue = "";
                                                        if (child.SelectToken("displayForm") != null)
                                                        {
                                                            metaValue = Regex.Replace(
                                                                child.SelectToken("displayForm").Value <string>(),
                                                                "<.*?>", string.Empty);
                                                            tag = await TagHandler.PrepareAttribute(context, metaName, metaValue, tagOrder++);
                                                            meta.Add(tag);
                                                        }
                                                        else
                                                        if (child.SelectToken("value") != null)
                                                        {
                                                            foreach (JToken value in child.SelectTokens("value").Children())
                                                            {
                                                                if (value.SelectToken("displayForm") != null)
                                                                {
                                                                    metaValue = Regex.Replace(
                                                                        value.SelectToken("displayForm").Value <string>(),
                                                                        "<.*?>", string.Empty);
                                                                    tag = await TagHandler.PrepareAttribute(context, metaName, metaValue, tagOrder++);
                                                                    meta.Add(tag);
                                                                }
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }

                                        string imageReproPageURL = "https://image01.cudl.lib.cam.ac.uk";


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = json;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }



                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "University of Cambridge Digital Library", 1);
                                        string viewerUrl    = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}";
                                        tag.ValueSupplement = viewerUrl;

                                        meta.Add(tag);

                                        book.Tags = meta.ToArray();

                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        int order = 0;
                                        foreach (JToken pageToken in parsed.SelectTokens("$.pages").Children())
                                        {
                                            order++;

                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.ProgressPercent = order;
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }

                                            string imageUrl = imageReproPageURL + pageToken.SelectToken("downloadImageURL").Value <string>();

                                            RArtifactItemRecord page = new RArtifactItemRecord()
                                            {
                                                Name                 = $"تصویر {order}",
                                                NameInEnglish        = $"Image {order}",
                                                Description          = "",
                                                DescriptionInEnglish = "",
                                                Order                = order,
                                                FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                LastModified         = DateTime.Now
                                            };

                                            List <RTagValue> pageMata = new List <RTagValue>();
                                            tag = await TagHandler.PrepareAttribute(context, "Source", "University of Cambridge Digital Library", 1);
                                            tag.ValueSupplement = $"{viewerUrl}/{order}";
                                            pageMata.Add(tag);

                                            if (pageToken.SelectToken("label") != null)
                                            {
                                                tag = await TagHandler.PrepareAttribute(context, "Label", pageToken.SelectToken("label").Value <string>(), 1);
                                                pageMata.Add(tag);
                                            }


                                            page.Tags = pageMata.ToArray();

                                            if (!string.IsNullOrEmpty(imageUrl))
                                            {
                                                bool recovered = false;
                                                if (
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    )
                                                {
                                                    RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                        imageUrl,
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                    if (picture.Result != null)
                                                    {
                                                        recovered            = true;
                                                        page.Images          = new RPictureFile[] { picture.Result };
                                                        page.CoverImageIndex = 0;

                                                        if (book.CoverItemIndex == (order - 1))
                                                        {
                                                            book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                        }

                                                        pages.Add(page);
                                                    }
                                                }
                                                if (!recovered)
                                                {
                                                    if (
                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )
                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }

                                                    var imageResult = await client.GetAsync(imageUrl);

                                                    if (imageResult.StatusCode == HttpStatusCode.NotFound)
                                                    {
                                                        break;        //finished
                                                    }


                                                    int _ImportRetryCount        = 200;
                                                    int _ImportRetryInitialSleep = 500;
                                                    int retryCount = 0;
                                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                    {
                                                        imageResult.Dispose();
                                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                        imageResult = await client.GetAsync(imageUrl);
                                                        retryCount++;
                                                    }

                                                    if (imageResult.IsSuccessStatusCode)
                                                    {
                                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                        {
                                                            RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                            if (picture.Result == null)
                                                            {
                                                                throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                            }



                                                            page.Images          = new RPictureFile[] { picture.Result };
                                                            page.CoverImageIndex = 0;

                                                            if (book.CoverItemIndex == (order - 1))
                                                            {
                                                                book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                            }

                                                            pages.Add(page);
                                                        }
                                                    }
                                                    else
                                                    {
                                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                                        {
                                                            job.EndTime   = DateTime.Now;
                                                            job.Status    = ImportJobStatus.Failed;
                                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                            importJobUpdaterDb.Update(job);
                                                            await importJobUpdaterDb.SaveChangesAsync();
                                                        }

                                                        imageResult.Dispose();
                                                        return;
                                                    }

                                                    imageResult.Dispose();
                                                    GC.Collect();
                                                }
                                            }
                                        }


                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (pages.Count == 0)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "Pages.Count == 0";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );



                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
        /// <summary>
        /// import from server folder
        /// </summary>
        /// <param name="folderPath">C:\Tools\batches\florence</param>
        /// <param name="friendlyUrl">shahname-florence</param>
        /// <param name="srcUrl">https://t.me/dr_khatibi_abolfazl/888</param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromServerFolder(string folderPath, string friendlyUrl, string srcUrl)
        {
            try
            {
                if (
                    (
                        await _context.ImportJobs
                        .Where(j => j.JobType == JobType.ServerFolder && j.ResourceNumber == folderPath && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                        .SingleOrDefaultAsync()
                    )
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing server folder {folderPath}"));
                }

                if (string.IsNullOrEmpty(friendlyUrl))
                {
                    return(new RServiceResult <bool>(false, $"Friendly url is empty, server folder {folderPath}"));
                }

                if (
                    (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
                }

                ImportJob job = new ImportJob()
                {
                    JobType         = JobType.ServerFolder,
                    ResourceNumber  = folderPath,
                    FriendlyUrl     = friendlyUrl,
                    SrcUrl          = srcUrl,
                    QueueTime       = DateTime.Now,
                    ProgressPercent = 0,
                    Status          = ImportJobStatus.NotStarted
                };


                await _context.ImportJobs.AddAsync
                (
                    job
                );

                await _context.SaveChangesAsync();

                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (RMuseumDbContext context = new RMuseumDbContext(Configuration))
                        {
                            RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from server folder {job.ResourceNumber}", $"extracted from server folder {job.ResourceNumber}")
                            {
                                Status         = PublishStatus.Draft,
                                DateTime       = DateTime.Now,
                                LastModified   = DateTime.Now,
                                CoverItemIndex = 0,
                                FriendlyUrl    = friendlyUrl
                            };


                            List <RTagValue> meta = new List <RTagValue>();
                            RTagValue tag;


                            tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                            meta.Add(tag);

                            tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                            meta.Add(tag);



                            meta.Add(tag);

                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                            {
                                job.StartTime  = DateTime.Now;
                                job.Status     = ImportJobStatus.Running;
                                job.SrcContent = "";
                                importJobUpdaterDb.Update(job);
                                await importJobUpdaterDb.SaveChangesAsync();
                            }

                            List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();
                            string[] fileNames = Directory.GetFiles(job.ResourceNumber, "*.jpg");
                            int order          = 0;
                            foreach (string fileName in fileNames)
                            {
                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                {
                                    job.ProgressPercent = order * 100 / (decimal)fileNames.Length;
                                    importJobUpdaterDb.Update(job);
                                    await importJobUpdaterDb.SaveChangesAsync();
                                }

                                order++;

                                RArtifactItemRecord page = new RArtifactItemRecord()
                                {
                                    Name                 = $"تصویر {order}",
                                    NameInEnglish        = $"Image {order} of {book.NameInEnglish}",
                                    Description          = "",
                                    DescriptionInEnglish = "",
                                    Order                = order,
                                    FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                    LastModified         = DateTime.Now
                                };


                                page.Tags = new RTagValue[] { };

                                if (
                                    File.Exists
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    )

                                    )
                                {
                                    File.Delete
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    );
                                }
                                if (

                                    File.Exists
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    )

                                    )
                                {
                                    File.Delete
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    );
                                }
                                if (

                                    File.Exists
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    )
                                    )
                                {
                                    File.Delete
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    );
                                }
                                using (FileStream imageStream = new FileStream(fileName, FileMode.Open))
                                {
                                    RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, job.SrcUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                    if (picture.Result == null)
                                    {
                                        throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                    }

                                    page.Images = new RPictureFile[] { picture.Result };

                                    page.CoverImageIndex = 0;

                                    if (book.CoverItemIndex == (order - 1))
                                    {
                                        book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                    }
                                }


                                pages.Add(page);
                            }


                            book.Tags = meta.ToArray();

                            book.Items     = pages.ToArray();
                            book.ItemCount = pages.Count;

                            if (pages.Count == 0)
                            {
                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                {
                                    job.EndTime   = DateTime.Now;
                                    job.Status    = ImportJobStatus.Failed;
                                    job.Exception = "Pages.Count == 0";
                                    importJobUpdaterDb.Update(job);
                                    await importJobUpdaterDb.SaveChangesAsync();
                                }
                                return;
                            }

                            await context.Artifacts.AddAsync(book);
                            await context.SaveChangesAsync();

                            job.ProgressPercent = 100;
                            job.Status          = ImportJobStatus.Succeeded;
                            job.ArtifactId      = book.Id;
                            job.EndTime         = DateTime.Now;
                            context.Update(job);
                            await context.SaveChangesAsync();
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );

                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
Example #3
0
        /// <summary>
        /// from http://www.thedigitalwalters.org/01_ACCESS_WALTERS_MANUSCRIPTS.html
        /// </summary>
        /// <param name="resourceNumber">W619</param>
        /// <param name="friendlyUrl">golestan-walters-01</param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromWalters(string resourceNumber, string friendlyUrl)
        {
            string url = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/ManuscriptDescriptions/{resourceNumber}_tei.xml";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.Walters && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
            }

            ImportJob job = new ImportJob()
            {
                JobType         = JobType.Walters,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();


            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string xml = await result.Content.ReadAsStringAsync();


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = xml;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }

                                        XElement elObject = XDocument.Parse(xml).Root;

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);



                                        try
                                        {
                                            foreach (var prop in elObject
                                                     .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}title"))
                                            {
                                                string label         = prop.Value;
                                                book.Name            = book.NameInEnglish =
                                                    book.Description = book.DescriptionInEnglish =
                                                        label;

                                                tag = await TagHandler.PrepareAttribute(context, "Title", label, 1);
                                                meta.Add(tag);
                                                break;
                                            }
                                        }
                                        catch
                                        {
                                            //ignore non-existing = null tags
                                        }

                                        try
                                        {
                                            foreach (var prop in elObject
                                                     .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}author"))
                                            {
                                                string label = prop.Value;
                                                tag          = await TagHandler.PrepareAttribute(context, "Contributor Names", label, 1);
                                                meta.Add(tag);
                                                break;
                                            }
                                        }
                                        catch
                                        {
                                            //ignore non-existing = null tags
                                        }

                                        try
                                        {
                                            foreach (var prop in elObject
                                                     .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}respStmt"))
                                            {
                                                string label = prop.Elements("{http://www.tei-c.org/ns/1.0}name").First().Value;
                                                tag          = await TagHandler.PrepareAttribute(context, "Contributor Names", label, 1);
                                                meta.Add(tag);
                                                break;
                                            }
                                        }
                                        catch
                                        {
                                            //ignore non-existing = null tags
                                        }

                                        try
                                        {
                                            foreach (var prop in elObject
                                                     .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}notesStmt").First()
                                                     .Elements("{http://www.tei-c.org/ns/1.0}note"))
                                            {
                                                string label = prop.Value;
                                                tag          = await TagHandler.PrepareAttribute(context, "Notes", label, 1);
                                                meta.Add(tag);
                                            }
                                        }
                                        catch
                                        {
                                            //ignore non-existing = null tags
                                        }



                                        tag = await TagHandler.PrepareAttribute(context, "Source", "Digitized Walters Manuscripts", 1);
                                        tag.ValueSupplement = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/html/{job.ResourceNumber}/";

                                        meta.Add(tag);


                                        book.Tags = meta.ToArray();
                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        int order = 0;

                                        foreach (var surface in elObject
                                                 .Elements("{http://www.tei-c.org/ns/1.0}facsimile").First()
                                                 .Elements("{http://www.tei-c.org/ns/1.0}surface"))
                                        {
                                            foreach (var graphic in surface.Elements("{http://www.tei-c.org/ns/1.0}graphic"))
                                            {
                                                if (graphic.Attribute("url").Value.Contains("sap.jpg"))
                                                {
                                                    order++;


                                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                    {
                                                        job.ProgressPercent = order;
                                                        importJobUpdaterDb.Update(job);
                                                        await importJobUpdaterDb.SaveChangesAsync();
                                                    }

                                                    RArtifactItemRecord page = new RArtifactItemRecord()
                                                    {
                                                        Name                 = $"تصویر {order}",
                                                        NameInEnglish        = $"Image {order}",
                                                        Description          = "",
                                                        DescriptionInEnglish = "",
                                                        Order                = order,
                                                        FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                        LastModified         = DateTime.Now
                                                    };



                                                    string imageUrl = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/{resourceNumber}/data/W.{resourceNumber.Substring(1)}/{graphic.Attribute("url").Value}";

                                                    tag = await TagHandler.PrepareAttribute(context, "Source", "Digitized Walters Manuscripts", 1);
                                                    tag.ValueSupplement = imageUrl;
                                                    page.Tags           = new RTagValue[] { tag };

                                                    if (!string.IsNullOrEmpty(imageUrl))
                                                    {
                                                        bool recovered = false;
                                                        if (
                                                            File.Exists
                                                            (
                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                            )
                                                            &&
                                                            File.Exists
                                                            (
                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                            )
                                                            &&
                                                            File.Exists
                                                            (
                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                            )
                                                            )
                                                        {
                                                            RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                                imageUrl,
                                                                                                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                                $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                            if (picture.Result != null)
                                                            {
                                                                recovered            = true;
                                                                page.Images          = new RPictureFile[] { picture.Result };
                                                                page.CoverImageIndex = 0;

                                                                if (book.CoverItemIndex == (order - 1))
                                                                {
                                                                    book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                                }

                                                                pages.Add(page);
                                                            }
                                                        }
                                                        if (!recovered)
                                                        {
                                                            if (
                                                                File.Exists
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                )

                                                                )
                                                            {
                                                                File.Delete
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                );
                                                            }
                                                            if (

                                                                File.Exists
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                )

                                                                )
                                                            {
                                                                File.Delete
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                );
                                                            }
                                                            if (

                                                                File.Exists
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                )
                                                                )
                                                            {
                                                                File.Delete
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                );
                                                            }
                                                            var imageResult = await client.GetAsync(imageUrl);


                                                            int _ImportRetryCount        = 5;
                                                            int _ImportRetryInitialSleep = 500;
                                                            int retryCount = 0;
                                                            while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                            {
                                                                imageResult.Dispose();
                                                                Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                                imageResult = await client.GetAsync(imageUrl);
                                                                retryCount++;
                                                            }

                                                            if (imageResult.IsSuccessStatusCode)
                                                            {
                                                                using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                                {
                                                                    RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                                    if (picture.Result == null)
                                                                    {
                                                                        throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                                    }

                                                                    page.Images          = new RPictureFile[] { picture.Result };
                                                                    page.CoverImageIndex = 0;

                                                                    if (book.CoverItemIndex == (order - 1))
                                                                    {
                                                                        book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                                    }

                                                                    pages.Add(page);
                                                                }
                                                            }
                                                            else
                                                            {
                                                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                                {
                                                                    job.EndTime   = DateTime.Now;
                                                                    job.Status    = ImportJobStatus.Failed;
                                                                    job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                                    importJobUpdaterDb.Update(job);
                                                                    await importJobUpdaterDb.SaveChangesAsync();
                                                                }

                                                                imageResult.Dispose();
                                                                return;
                                                            }

                                                            imageResult.Dispose();
                                                            GC.Collect();
                                                        }
                                                    }
                                                }
                                            }
                                        }



                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (pages.Count == 0)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "Pages.Count == 0";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );



                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
        private async Task <RServiceResult <List <RArtifactItemRecord> > > _InternalHarvardJsonImport(string hardvardResourceNumber, ImportJob job, string friendlyUrl, RMuseumDbContext context, RArtifactMasterRecord book, List <RTagValue> meta)
        {
            List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

            using (var client = new HttpClient())
            {
                using (var jsonResult = await client.GetAsync($"https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}"))
                {
                    if (jsonResult.IsSuccessStatusCode)
                    {
                        string json = await jsonResult.Content.ReadAsStringAsync();

                        var parsed = JObject.Parse(json);
                        book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish =
                            parsed.SelectToken("label").Value <string>();


                        RTagValue tag;

                        tag = await TagHandler.PrepareAttribute(context, "Title", book.Name, 1);

                        meta.Add(tag);

                        tag = await TagHandler.PrepareAttribute(context, "Contributor Names", "تعیین نشده", 1);

                        meta.Add(tag);

                        List <string> labels = new List <string>();
                        foreach (JToken structure in parsed.SelectTokens("$.structures[*].label"))
                        {
                            labels.Add(structure.Value <string>());
                        }

                        int order     = 0;
                        var canvases  = parsed.SelectToken("sequences").First().SelectToken("canvases").ToArray();
                        int pageCount = canvases.Length;
                        foreach (JToken canvas in canvases)
                        {
                            order++;
                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                            {
                                job.ProgressPercent = order * 100 / (decimal)pageCount;
                                importJobUpdaterDb.Update(job);
                                await importJobUpdaterDb.SaveChangesAsync();
                            }
                            string label = canvas.SelectToken("label").Value <string>();
                            if (labels.Where(l => l.IndexOf(label) != -1).SingleOrDefault() != null)
                            {
                                label = labels.Where(l => l.IndexOf(label) != -1).SingleOrDefault();
                            }
                            string imageUrl          = canvas.SelectTokens("images[*]").First().SelectToken("resource").SelectToken("@id").Value <string>();
                            RArtifactItemRecord page = new RArtifactItemRecord()
                            {
                                Name                 = $"تصویر {order}",
                                NameInEnglish        = label,
                                Description          = "",
                                DescriptionInEnglish = "",
                                Order                = order,
                                FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                LastModified         = DateTime.Now
                            };



                            tag = await TagHandler.PrepareAttribute(context, "Source", "Harvard University Islamic Heritage Project", 1);

                            tag.ValueSupplement = imageUrl;
                            page.Tags           = new RTagValue[] { tag };


                            if (!string.IsNullOrEmpty(imageUrl))
                            {
                                bool recovered = false;
                                if (
                                    File.Exists
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    )
                                    &&
                                    File.Exists
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    )
                                    &&
                                    File.Exists
                                    (
                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                    )
                                    )
                                {
                                    RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                        imageUrl,
                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                        $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);

                                    if (picture.Result != null)
                                    {
                                        recovered            = true;
                                        page.Images          = new RPictureFile[] { picture.Result };
                                        page.CoverImageIndex = 0;

                                        if (book.CoverItemIndex == (order - 1))
                                        {
                                            book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                        }

                                        pages.Add(page);
                                    }
                                }
                                if (!recovered)
                                {
                                    if (
                                        File.Exists
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        )

                                        )
                                    {
                                        File.Delete
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        );
                                    }
                                    if (

                                        File.Exists
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        )

                                        )
                                    {
                                        File.Delete
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        );
                                    }
                                    if (

                                        File.Exists
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        )
                                        )
                                    {
                                        File.Delete
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        );
                                    }
                                    var imageResult = await client.GetAsync(imageUrl);


                                    int _ImportRetryCount        = 5;
                                    int _ImportRetryInitialSleep = 500;
                                    int retryCount = 0;
                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                    {
                                        imageResult.Dispose();
                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                        imageResult = await client.GetAsync(imageUrl);

                                        retryCount++;
                                    }

                                    if (imageResult.IsSuccessStatusCode)
                                    {
                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                        {
                                            RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);

                                            if (picture.Result == null)
                                            {
                                                throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                            }

                                            page.Images          = new RPictureFile[] { picture.Result };
                                            page.CoverImageIndex = 0;

                                            if (book.CoverItemIndex == (order - 1))
                                            {
                                                book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                            }

                                            pages.Add(page);
                                        }
                                    }
                                    else
                                    {
                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                        {
                                            job.EndTime   = DateTime.Now;
                                            job.Status    = ImportJobStatus.Failed;
                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }

                                        imageResult.Dispose();
                                        return(new RServiceResult <List <RArtifactItemRecord> >(null, "failed"));
                                    }

                                    imageResult.Dispose();
                                    GC.Collect();
                                }
                            }
                        }
                    }
                    else
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = $"Http result is not ok ({jsonResult.StatusCode}) for https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}";
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                        return(new RServiceResult <List <RArtifactItemRecord> >(null, "failed"));
                    }
                }
            }

            return(new RServiceResult <List <RArtifactItemRecord> >(pages));
        }
        /// <summary>
        /// from https://curiosity.lib.harvard.edu
        /// </summary>
        /// <param name="url">example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941</param>
        /// <param name="friendlyUrl"></param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromHarvard(string url, string friendlyUrl)
        {
            try
            {
                if (
                    (
                        await _context.ImportJobs
                        .Where(j => j.JobType == JobType.Harvard && j.ResourceNumber == url && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                        .SingleOrDefaultAsync()
                    )
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
                }

                if (string.IsNullOrEmpty(friendlyUrl))
                {
                    return(new RServiceResult <bool>(false, $"Friendly url is empty, url = {url}"));
                }

                if (
                    (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
                }

                ImportJob job = new ImportJob()
                {
                    JobType         = JobType.Harvard,
                    ResourceNumber  = url,
                    FriendlyUrl     = friendlyUrl,
                    SrcUrl          = url,
                    QueueTime       = DateTime.Now,
                    ProgressPercent = 0,
                    Status          = ImportJobStatus.NotStarted
                };


                await _context.ImportJobs.AddAsync
                (
                    job
                );

                await _context.SaveChangesAsync();

                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string html = await result.Content.ReadAsStringAsync();


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = html;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }

                                        int nStartIndex = html.IndexOf("<dt");
                                        while (nStartIndex != -1)
                                        {
                                            nStartIndex = html.IndexOf(">", nStartIndex);
                                            if (nStartIndex == -1)
                                            {
                                                break;
                                            }
                                            nStartIndex++;
                                            string tagName = html.Substring(nStartIndex, html.IndexOf(":", nStartIndex) - nStartIndex);
                                            nStartIndex    = html.IndexOf("<dd", nStartIndex);
                                            if (nStartIndex == -1)
                                            {
                                                break;
                                            }
                                            nStartIndex = html.IndexOf(">", nStartIndex);
                                            if (nStartIndex == -1)
                                            {
                                                break;
                                            }
                                            nStartIndex++;
                                            string tagValues = html.Substring(nStartIndex, html.IndexOf("</dd>", nStartIndex) - nStartIndex);
                                            foreach (string tagValuePart in tagValues.Split("<br/>", StringSplitOptions.RemoveEmptyEntries))
                                            {
                                                string tagValue = tagValuePart;
                                                bool href       = false;
                                                if (tagValue.IndexOf("<a href=") != -1)
                                                {
                                                    href     = true;
                                                    tagValue = tagValue.Substring(tagValue.IndexOf('>') + 1);
                                                    tagValue = tagValue.Substring(0, tagValue.IndexOf('<'));
                                                }
                                                tag = await TagHandler.PrepareAttribute(context, tagName, tagValue, 1);
                                                if (href)
                                                {
                                                    tag.ValueSupplement = tagValue;
                                                }
                                                meta.Add(tag);
                                            }

                                            nStartIndex = html.IndexOf("<dt", nStartIndex + 1);
                                        }



                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "Harvard University Islamic Heritage Project", 1);
                                        tag.ValueSupplement = $"{job.SrcUrl}";

                                        meta.Add(tag);

                                        nStartIndex = html.IndexOf("https://pds.lib.harvard.edu/pds/view/");
                                        if (nStartIndex == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "Not found https://pds.lib.harvard.edu/pds/view/";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nStartIndex += "https://pds.lib.harvard.edu/pds/view/".Length;

                                        string hardvardResourceNumber = html.Substring(nStartIndex, html.IndexOf('\"', nStartIndex) - nStartIndex);

                                        List <RArtifactItemRecord> pages = (await _InternalHarvardJsonImport(hardvardResourceNumber, job, friendlyUrl, context, book, meta)).Result;
                                        if (pages == null)
                                        {
                                            return;
                                        }


                                        book.Tags = meta.ToArray();

                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (pages.Count == 0)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "Pages.Count == 0";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );

                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
Example #6
0
        /// <summary>
        /// from https://catalog.hathitrust.org
        /// </summary>
        /// <param name="resourceNumber">006814127</param>
        /// <param name="friendlyUrl"></param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromHathiTrust(string resourceNumber, string friendlyUrl)
        {
            string url = $"https://catalog.hathitrust.org/Record/{resourceNumber}.xml";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.HathiTrust && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated artifact friendly url '{friendlyUrl}'"));
            }



            ImportJob job = new ImportJob()
            {
                JobType         = JobType.HathiTrust,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();


            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        if (
                                            (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync())
                                            !=
                                            null
                                            )
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "aborted because of duplicated friendly url";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string xml = await result.Content.ReadAsStringAsync();


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = xml;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }

                                        string title             = "";
                                        string author            = "";
                                        string pdfResourceNumber = "";
                                        int tagOrder             = 1;
                                        XElement elObject        = XDocument.Parse(xml).Root;
                                        foreach (var datafield in elObject.Element("record").Elements("datafield"))
                                        {
                                            tagOrder++;
                                            if (datafield.Attribute("tag") == null)
                                            {
                                                continue;
                                            }
                                            string hathiTrustTag = datafield.Attribute("tag").Value;
                                            switch (hathiTrustTag)
                                            {
                                            case "245":
                                            case "246":
                                                foreach (var subfield in datafield.Elements("subfield"))
                                                {
                                                    if (subfield.Attribute("code") != null)
                                                    {
                                                        if (subfield.Attribute("code").Value == "a" || subfield.Attribute("code").Value == "f")
                                                        {
                                                            title = (title + " " + subfield.Value).Trim();
                                                        }
                                                    }
                                                }
                                                break;

                                            case "100":
                                                foreach (var subfield in datafield.Elements("subfield"))
                                                {
                                                    if (subfield.Attribute("code") != null)
                                                    {
                                                        if (subfield.Attribute("code").Value == "a" || subfield.Attribute("code").Value == "d")
                                                        {
                                                            author = (author + " " + subfield.Value).Trim();
                                                        }
                                                    }
                                                }
                                                break;

                                            case "HOL":
                                                foreach (var subfield in datafield.Elements("subfield"))
                                                {
                                                    if (subfield.Attribute("code") != null)
                                                    {
                                                        if (subfield.Attribute("code").Value == "p")
                                                        {
                                                            pdfResourceNumber = subfield.Value;
                                                        }
                                                    }
                                                }
                                                break;

                                            default:
                                                {
                                                    if (int.TryParse(hathiTrustTag, out int tmp))
                                                    {
                                                        if (tmp >= 100 && tmp <= 900)
                                                        {
                                                            string note = "";
                                                            foreach (var subfield in datafield.Elements("subfield"))
                                                            {
                                                                if (subfield.Attribute("code") != null)
                                                                {
                                                                    note = (note + " " + subfield.Value).Trim();
                                                                }
                                                            }

                                                            tag = await TagHandler.PrepareAttribute(context, "Notes", note, tagOrder);
                                                            meta.Add(tag);
                                                        }
                                                    }
                                                }
                                                break;
                                            }
                                        }

                                        if (string.IsNullOrEmpty(pdfResourceNumber))
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "pdfResourceNumber not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        tag = await TagHandler.PrepareAttribute(context, "Title", title, 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Contributor Names", author, 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "HathiTrust Digital Library", 1);
                                        string viewerUrl    = $"https://babel.hathitrust.org/cgi/pt?id={pdfResourceNumber}";
                                        tag.ValueSupplement = viewerUrl;

                                        meta.Add(tag);

                                        book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = title;
                                        book.Tags = meta.ToArray();

                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        string lastMD5hash = "";
                                        int order          = 0;
                                        while (true)
                                        {
                                            order++;
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.ProgressPercent = order;
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }

                                            string imageUrl          = $"https://babel.hathitrust.org/cgi/imgsrv/image?id={pdfResourceNumber};seq={order};size=1000;rotation=0";
                                            RArtifactItemRecord page = new RArtifactItemRecord()
                                            {
                                                Name                 = $"تصویر {order}",
                                                NameInEnglish        = $"Image {order}",
                                                Description          = "",
                                                DescriptionInEnglish = "",
                                                Order                = order,
                                                FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                LastModified         = DateTime.Now
                                            };

                                            tag = await TagHandler.PrepareAttribute(context, "Source", "HathiTrust Digital Library", 1);
                                            tag.ValueSupplement = viewerUrl;
                                            page.Tags           = new RTagValue[] { tag };

                                            if (!string.IsNullOrEmpty(imageUrl))
                                            {
                                                bool recovered = false;
                                                if (
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    )
                                                {
                                                    RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                        imageUrl,
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                    if (picture.Result != null)
                                                    {
                                                        recovered            = true;
                                                        page.Images          = new RPictureFile[] { picture.Result };
                                                        page.CoverImageIndex = 0;

                                                        if (book.CoverItemIndex == (order - 1))
                                                        {
                                                            book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                        }

                                                        pages.Add(page);
                                                    }
                                                }
                                                if (!recovered)
                                                {
                                                    if (
                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )
                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    var imageResult = await client.GetAsync(imageUrl);


                                                    int _ImportRetryCount        = 200;
                                                    int _ImportRetryInitialSleep = 500;
                                                    int retryCount = 0;
                                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                    {
                                                        imageResult.Dispose();
                                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                        imageResult = await client.GetAsync(imageUrl);
                                                        retryCount++;
                                                    }

                                                    if (imageResult.IsSuccessStatusCode)
                                                    {
                                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                        {
                                                            RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                            if (picture.Result == null)
                                                            {
                                                                throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                            }

                                                            bool lastPage = false;
                                                            using (var md5 = MD5.Create())
                                                            {
                                                                string md5hash = string.Join("", md5.ComputeHash(File.ReadAllBytes(Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"))).Select(x => x.ToString("X2")));
                                                                if (md5hash == lastMD5hash)
                                                                {
                                                                    File.Delete
                                                                    (
                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                    );
                                                                    File.Delete
                                                                    (
                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                    );
                                                                    File.Delete
                                                                    (
                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                    );
                                                                    lastPage = true;
                                                                }
                                                                lastMD5hash = md5hash;
                                                            }

                                                            if (!lastPage)
                                                            {
                                                                page.Images          = new RPictureFile[] { picture.Result };
                                                                page.CoverImageIndex = 0;

                                                                if (book.CoverItemIndex == (order - 1))
                                                                {
                                                                    book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                                }

                                                                pages.Add(page);
                                                            }
                                                            else
                                                            {
                                                                break;
                                                            }
                                                        }
                                                    }
                                                    else
                                                    {
                                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                        {
                                                            job.EndTime   = DateTime.Now;
                                                            job.Status    = ImportJobStatus.Failed;
                                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                            importJobUpdaterDb.Update(job);
                                                            await importJobUpdaterDb.SaveChangesAsync();
                                                        }

                                                        imageResult.Dispose();
                                                        return;
                                                    }

                                                    imageResult.Dispose();
                                                    GC.Collect();
                                                }
                                            }
                                        }


                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (pages.Count == 0)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "ages.Count == 0";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );



                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
        /// <summary>
        /// import from https://viewer.cbl.ie
        /// </summary>
        /// <param name="resourceNumber">119</param>
        /// <param name="friendlyUrl">golestan-baysonghori</param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromChesterBeatty(string resourceNumber, string friendlyUrl)
        {
            try
            {
                string srcUrl = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/1/";
                if (
                    (
                        await _context.ImportJobs
                        .Where(j => j.JobType == JobType.ChesterBeatty && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                        .SingleOrDefaultAsync()
                    )
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {srcUrl}"));
                }

                if (string.IsNullOrEmpty(friendlyUrl))
                {
                    return(new RServiceResult <bool>(false, $"Friendly url is empty, server folder {srcUrl}"));
                }

                if (
                    (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
                }

                ImportJob job = new ImportJob()
                {
                    JobType         = JobType.ChesterBeatty,
                    ResourceNumber  = resourceNumber,
                    FriendlyUrl     = friendlyUrl,
                    SrcUrl          = srcUrl,
                    QueueTime       = DateTime.Now,
                    ProgressPercent = 0,
                    Status          = ImportJobStatus.NotStarted
                };


                await _context.ImportJobs.AddAsync
                (
                    job
                );

                await _context.SaveChangesAsync();

                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (RMuseumDbContext context = new RMuseumDbContext(Configuration))
                        {
                            RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from url {job.ResourceNumber}", $"extracted from url {job.ResourceNumber}")
                            {
                                Status         = PublishStatus.Draft,
                                DateTime       = DateTime.Now,
                                LastModified   = DateTime.Now,
                                CoverItemIndex = 0,
                                FriendlyUrl    = friendlyUrl,
                            };


                            List <RTagValue> meta = new List <RTagValue>();
                            RTagValue tag;


                            tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                            meta.Add(tag);

                            tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                            meta.Add(tag);

                            tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1);
                            tag.ValueSupplement = srcUrl;

                            meta.Add(tag);



                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                            {
                                job.StartTime  = DateTime.Now;
                                job.Status     = ImportJobStatus.Running;
                                job.SrcContent = "";
                                importJobUpdaterDb.Update(job);
                                await importJobUpdaterDb.SaveChangesAsync();
                            }

                            List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();
                            int order = 0;
                            using (var client = new HttpClient())
                                do
                                {
                                    order++;

                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                    {
                                        job.ProgressPercent = order;
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }

                                    RArtifactItemRecord page = new RArtifactItemRecord()
                                    {
                                        Name                 = $"تصویر {order}",
                                        NameInEnglish        = $"Image {order} of {book.NameInEnglish}",
                                        Description          = "",
                                        DescriptionInEnglish = "",
                                        Order                = order,
                                        FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                        LastModified         = DateTime.Now
                                    };

                                    string imageUrl = $"https://viewer.cbl.ie/viewer/rest/image/Per_{resourceNumber}/Per{resourceNumber}_{$"{order}".PadLeft(3, '0')}.jpg/full/!10000,10000/0/default.jpg?ignoreWatermark=true";


                                    page.Tags = new RTagValue[] { };

                                    bool recovered = false;

                                    if (
                                        File.Exists
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        )
                                        &&
                                        File.Exists
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        )
                                        &&
                                        File.Exists
                                        (
                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                        )
                                        )
                                    {
                                        RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                            imageUrl,
                                                                                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                            $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                        if (picture.Result != null)
                                        {
                                            recovered            = true;
                                            page.Images          = new RPictureFile[] { picture.Result };
                                            page.CoverImageIndex = 0;

                                            if (book.CoverItemIndex == (order - 1))
                                            {
                                                book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                            }

                                            tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1);
                                            tag.ValueSupplement = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/{$"{order}".PadLeft(3, '0')}/";;
                                            page.Tags           = new RTagValue[] { tag };

                                            pages.Add(page);
                                        }
                                    }

                                    if (!recovered)
                                    {
                                        if (
                                            File.Exists
                                            (
                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                            )

                                            )
                                        {
                                            File.Delete
                                            (
                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                            );
                                        }
                                        if (

                                            File.Exists
                                            (
                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                            )

                                            )
                                        {
                                            File.Delete
                                            (
                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                            );
                                        }
                                        if (

                                            File.Exists
                                            (
                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                            )
                                            )
                                        {
                                            File.Delete
                                            (
                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                            );
                                        }
                                        var imageResult = await client.GetAsync(imageUrl);

                                        if (imageResult.StatusCode == HttpStatusCode.Forbidden || imageResult.StatusCode == HttpStatusCode.NotFound)
                                        {
                                            break;
                                        }


                                        int _ImportRetryCount        = 5;
                                        int _ImportRetryInitialSleep = 500;
                                        int retryCount = 0;
                                        while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                        {
                                            imageResult.Dispose();
                                            Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                            imageResult = await client.GetAsync(imageUrl);
                                            retryCount++;
                                        }

                                        if (imageResult.IsSuccessStatusCode)
                                        {
                                            using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                            {
                                                RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                if (picture.Result == null)
                                                {
                                                    throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                }

                                                page.Images          = new RPictureFile[] { picture.Result };
                                                page.CoverImageIndex = 0;

                                                if (book.CoverItemIndex == (order - 1))
                                                {
                                                    book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                }
                                                tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1);
                                                tag.ValueSupplement = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/{$"{order}".PadLeft(3, '0')}/";
                                                page.Tags           = new RTagValue[] { tag };

                                                pages.Add(page);
                                            }
                                        }
                                        else
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }

                                            imageResult.Dispose();
                                            return;
                                        }


                                        imageResult.Dispose();
                                        GC.Collect();
                                    }


                                    pages.Add(page);
                                }while (true);


                            book.Tags = meta.ToArray();

                            book.Items     = pages.ToArray();
                            book.ItemCount = pages.Count;

                            if (pages.Count == 0)
                            {
                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                {
                                    job.EndTime   = DateTime.Now;
                                    job.Status    = ImportJobStatus.Failed;
                                    job.Exception = "Pages.Count == 0";
                                    importJobUpdaterDb.Update(job);
                                    await importJobUpdaterDb.SaveChangesAsync();
                                }
                                return;
                            }

                            await context.Artifacts.AddAsync(book);
                            await context.SaveChangesAsync();

                            job.ProgressPercent = 100;
                            job.Status          = ImportJobStatus.Succeeded;
                            job.ArtifactId      = book.Id;
                            job.EndTime         = DateTime.Now;
                            context.Update(job);
                            await context.SaveChangesAsync();
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );

                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
        /// <summary>
        /// import from http://www.qajarwomen.org
        /// </summary>
        /// <param name="hardvardResourceNumber">43117279</param>
        /// <param name="friendlyUrl">atame</param>
        /// <param name="srcUrl">http://www.qajarwomen.org/fa/items/1018A10.html</param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromHarvardDirectly(string hardvardResourceNumber, string friendlyUrl, string srcUrl)
        {
            try
            {
                if (
                    (
                        await _context.ImportJobs
                        .Where(j => j.JobType == JobType.HarvardDirect && j.ResourceNumber == hardvardResourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                        .SingleOrDefaultAsync()
                    )
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing harvard direct resource number {hardvardResourceNumber}"));
                }

                if (string.IsNullOrEmpty(friendlyUrl))
                {
                    return(new RServiceResult <bool>(false, $"Friendly url is empty, harvard direct resource number {hardvardResourceNumber}"));
                }

                if (
                    (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                    !=
                    null
                    )
                {
                    return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
                }

                ImportJob job = new ImportJob()
                {
                    JobType         = JobType.HarvardDirect,
                    ResourceNumber  = hardvardResourceNumber,
                    FriendlyUrl     = friendlyUrl,
                    SrcUrl          = srcUrl,
                    QueueTime       = DateTime.Now,
                    ProgressPercent = 0,
                    Status          = ImportJobStatus.NotStarted
                };


                await _context.ImportJobs.AddAsync
                (
                    job
                );

                await _context.SaveChangesAsync();

                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from harvard resource number {job.ResourceNumber}", $"extracted from harvard resource number {job.ResourceNumber}")
                            {
                                Status         = PublishStatus.Draft,
                                DateTime       = DateTime.Now,
                                LastModified   = DateTime.Now,
                                CoverItemIndex = 0,
                                FriendlyUrl    = friendlyUrl
                            };


                            List <RTagValue> meta = new List <RTagValue>();
                            RTagValue tag;



                            tag = await TagHandler.PrepareAttribute(context, "Notes", "وارد شده از سایت دنیای زنان در عصر قاجار", 1);
                            meta.Add(tag);



                            tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                            meta.Add(tag);

                            tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                            meta.Add(tag);


                            tag = await TagHandler.PrepareAttribute(context, "Source", "دنیای زنان در عصر قاجار", 1);
                            tag.ValueSupplement = $"{job.SrcUrl}";

                            meta.Add(tag);

                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                            {
                                job.StartTime  = DateTime.Now;
                                job.Status     = ImportJobStatus.Running;
                                job.SrcContent = $"https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}";
                                importJobUpdaterDb.Update(job);
                                await importJobUpdaterDb.SaveChangesAsync();
                            }

                            List <RArtifactItemRecord> pages = (await _InternalHarvardJsonImport(hardvardResourceNumber, job, friendlyUrl, context, book, meta)).Result;
                            if (pages == null)
                            {
                                return;
                            }


                            book.Tags = meta.ToArray();

                            book.Items     = pages.ToArray();
                            book.ItemCount = pages.Count;

                            if (pages.Count == 0)
                            {
                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                {
                                    job.EndTime   = DateTime.Now;
                                    job.Status    = ImportJobStatus.Failed;
                                    job.Exception = "Pages.Count == 0";
                                    importJobUpdaterDb.Update(job);
                                    await importJobUpdaterDb.SaveChangesAsync();
                                }
                                return;
                            }

                            await context.Artifacts.AddAsync(book);
                            await context.SaveChangesAsync();

                            job.ProgressPercent = 100;
                            job.Status          = ImportJobStatus.Succeeded;
                            job.ArtifactId      = book.Id;
                            job.EndTime         = DateTime.Now;
                            context.Update(job);
                            await context.SaveChangesAsync();
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );

                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
        /// <summary>
        /// from http://www.bl.uk
        /// </summary>
        /// <param name="resourceNumber">grenville_xli_f001r</param>
        /// <param name="friendlyUrl"></param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromBritishLibrary(string resourceNumber, string friendlyUrl)
        {
            string url = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={resourceNumber}";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.BritishLibrary && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated artifact friendly url '{friendlyUrl}'"));
            }



            ImportJob job = new ImportJob()
            {
                JobType         = JobType.BritishLibrary,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();


            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            client.Timeout = TimeSpan.FromMinutes(5);
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(Configuration))
                                    {
                                        if (
                                            (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync())
                                            !=
                                            null
                                            )
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "aborted because of duplicated friendly url";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string html = await result.Content.ReadAsStringAsync();


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = html;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }


                                        int nIdxStart = html.IndexOf("PageList");
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "PageList not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nIdxStart = html.IndexOf("value=\"", nIdxStart);
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "value after PageList not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nIdxStart += "value=\"".Length;

                                        string strPageList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart);

                                        nIdxStart = html.IndexOf("TextList");
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "TextList not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nIdxStart = html.IndexOf("value=\"", nIdxStart);
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "value after TextList not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nIdxStart += "value=\"".Length;

                                        string strTextList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart);

                                        nIdxStart = html.IndexOf("TitleList");
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "TitleList not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nIdxStart = html.IndexOf("value=\"", nIdxStart);
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "value after TitleList not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        nIdxStart += "value=\"".Length;

                                        string strTitleList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart);

                                        string[] PageUrls   = strPageList.Split("||", StringSplitOptions.None);
                                        string[] PageTexts  = strTextList.Split("||", StringSplitOptions.None);
                                        string[] PageTitles = strTitleList.Split("||", StringSplitOptions.None);

                                        if (PageUrls.Length != PageTexts.Length || PageTexts.Length != PageTitles.Length)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "PageUrls.Length != PageTexts.Length || PageTexts.Length != PageTitles.Length";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        tag = await TagHandler.PrepareAttribute(context, "Title", "Untitled", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Contributor Names", "Unknown", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);

                                        book.Tags = meta.ToArray();


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "British Library", 1);
                                        string viewerUrl    = $"http://www.bl.uk/manuscripts/FullDisplay.aspx?ref={resourceNumber.Substring(0, resourceNumber.LastIndexOf('_'))}";
                                        tag.ValueSupplement = viewerUrl;

                                        meta.Add(tag);

                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        int order = 0;
                                        for (int i = 0; i < PageUrls.Length; i++)
                                        {
                                            if (PageUrls[i] == "##")
                                            {
                                                continue;
                                            }
                                            order++;

                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.ProgressPercent = order;
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }


                                            RArtifactItemRecord page = new RArtifactItemRecord()
                                            {
                                                Name                 = $"تصویر {order}",
                                                NameInEnglish        = $"Image {order}",
                                                Description          = "",
                                                DescriptionInEnglish = "",
                                                Order                = order,
                                                FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                LastModified         = DateTime.Now
                                            };


                                            List <RTagValue> pageTags = new List <RTagValue>();
                                            tag = await TagHandler.PrepareAttribute(context, "Source", "British Library", 1);
                                            tag.ValueSupplement = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={PageUrls[i]}";
                                            pageTags.Add(tag);


                                            if (!string.IsNullOrEmpty(PageTitles[i]))
                                            {
                                                RTagValue toc       = await TagHandler.PrepareAttribute(context, "Title in TOC", PageTitles[i], 1);
                                                toc.ValueSupplement = "1";        //font size
                                                pageTags.Add(toc);
                                            }

                                            if (!string.IsNullOrEmpty(PageTexts[i]))
                                            {
                                                tag = await TagHandler.PrepareAttribute(context, "Label", PageTexts[i], 1);
                                                pageTags.Add(tag);
                                            }

                                            page.Tags = pageTags.ToArray();


                                            bool recovered = false;
                                            if (
                                                File.Exists
                                                (
                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                )
                                                &&
                                                File.Exists
                                                (
                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                )
                                                &&
                                                File.Exists
                                                (
                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                )
                                                )
                                            {
                                                RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                    viewerUrl,
                                                                                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                    $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                if (picture.Result != null)
                                                {
                                                    recovered            = true;
                                                    page.Images          = new RPictureFile[] { picture.Result };
                                                    page.CoverImageIndex = 0;

                                                    if (book.CoverItemIndex == (order - 1))
                                                    {
                                                        book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                    }

                                                    pages.Add(page);
                                                }
                                            }
                                            if (!recovered)
                                            {
                                                if (
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )

                                                    )
                                                {
                                                    File.Delete
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    );
                                                }
                                                if (

                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )

                                                    )
                                                {
                                                    File.Delete
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    );
                                                }
                                                if (

                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    )
                                                {
                                                    File.Delete
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    );
                                                }

                                                /*
                                                 * failed multithread attempt:
                                                 *
                                                 *  BLTileMixer mixer = new BLTileMixer();
                                                 *  RServiceResult<Stream> blResult = await mixer.DownloadMix(PageUrls[i], order);
                                                 */


                                                Dictionary <(int x, int y), Image> tiles = new Dictionary <(int x, int y), Image>();
                                                int max_x = -1;
                                                for (int x = 0; ; x++)
                                                {
                                                    string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/{x}_0.jpg";
                                                    var imageResult = await client.GetAsync(imageUrl);

                                                    int _ImportRetryCount        = 5;
                                                    int _ImportRetryInitialSleep = 500;
                                                    int retryCount = 0;
                                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                    {
                                                        imageResult.Dispose();
                                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                        imageResult = await client.GetAsync(imageUrl);
                                                        retryCount++;
                                                    }

                                                    if (imageResult.IsSuccessStatusCode)
                                                    {
                                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                        {
                                                            imageStream.Position = 0;
                                                            try
                                                            {
                                                                Image tile = Image.FromStream(imageStream);
                                                                tiles.Add((x, 0), tile);
                                                                max_x = x;
                                                            }
                                                            catch (Exception aexp)
                                                            {
                                                                if (aexp is ArgumentException)
                                                                {
                                                                    break;
                                                                }
                                                                throw aexp;
                                                            }
                                                        }
                                                    }
                                                    else
                                                    {
                                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                                        {
                                                            job.EndTime   = DateTime.Now;
                                                            job.Status    = ImportJobStatus.Failed;
                                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                            importJobUpdaterDb.Update(job);
                                                            await importJobUpdaterDb.SaveChangesAsync();
                                                        }

                                                        imageResult.Dispose();
                                                        return;
                                                    }
                                                }

                                                int max_y = -1;
                                                for (int y = 1; ; y++)
                                                {
                                                    string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/0_{y}.jpg";
                                                    var imageResult = await client.GetAsync(imageUrl);

                                                    int _ImportRetryCount        = 5;
                                                    int _ImportRetryInitialSleep = 500;
                                                    int retryCount = 0;
                                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                    {
                                                        imageResult.Dispose();
                                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                        imageResult = await client.GetAsync(imageUrl);
                                                        retryCount++;
                                                    }

                                                    if (imageResult.IsSuccessStatusCode)
                                                    {
                                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                        {
                                                            if (imageStream.Length <= 248)
                                                            {
                                                                break;
                                                            }
                                                            imageStream.Position = 0;
                                                            try
                                                            {
                                                                Image tile = Image.FromStream(imageStream);
                                                                tiles.Add((0, y), tile);
                                                                max_y = y;
                                                            }
                                                            catch (Exception aexp)
                                                            {
                                                                if (aexp is ArgumentException)
                                                                {
                                                                    break;
                                                                }
                                                                throw aexp;
                                                            }
                                                        }
                                                    }
                                                    else
                                                    {
                                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                                        {
                                                            job.EndTime   = DateTime.Now;
                                                            job.Status    = ImportJobStatus.Failed;
                                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                            importJobUpdaterDb.Update(job);
                                                            await importJobUpdaterDb.SaveChangesAsync();
                                                        }

                                                        imageResult.Dispose();
                                                        return;
                                                    }
                                                }

                                                for (int x = 0; x <= max_x; x++)
                                                {
                                                    for (int y = 0; y <= max_y; y++)
                                                    {
                                                        if (tiles.TryGetValue((x, y), out Image tmp) == false)
                                                        {
                                                            string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/{x}_{y}.jpg";
                                                            var imageResult = await client.GetAsync(imageUrl);

                                                            int _ImportRetryCount        = 5;
                                                            int _ImportRetryInitialSleep = 500;
                                                            int retryCount = 0;
                                                            while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                            {
                                                                imageResult.Dispose();
                                                                Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                                imageResult = await client.GetAsync(imageUrl);
                                                                retryCount++;
                                                            }

                                                            if (imageResult.IsSuccessStatusCode)
                                                            {
                                                                using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                                {
                                                                    if (imageStream.Length == 0)
                                                                    {
                                                                        break;
                                                                    }
                                                                    imageStream.Position = 0;
                                                                    tiles.Add((x, y), Image.FromStream(imageStream));
                                                                }
                                                            }
                                                            else
                                                            {
                                                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                                                {
                                                                    job.EndTime   = DateTime.Now;
                                                                    job.Status    = ImportJobStatus.Failed;
                                                                    job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                                    importJobUpdaterDb.Update(job);
                                                                    await importJobUpdaterDb.SaveChangesAsync();
                                                                }

                                                                imageResult.Dispose();
                                                                return;
                                                            }
                                                        }
                                                    }
                                                }

                                                int tileWidth  = tiles[(0, 0)].Width;
                                                int tileHeight = tiles[(0, 0)].Height;
        /// <summary>
        /// from http://pudl.princeton.edu/
        /// </summary>
        /// <param name="resourceNumber">dj52w476m</param>
        /// <param name="friendlyUrl"></param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromPrinceton(string resourceNumber, string friendlyUrl)
        {
            string url = $"http://pudl.princeton.edu/mdCompiler2.php?obj={resourceNumber}";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.Princeton && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
            }

            ImportJob job = new ImportJob()
            {
                JobType         = JobType.Princeton,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();


            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string xml = await result.Content.ReadAsStringAsync();


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = xml;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }

                                        XElement elObject = XDocument.Parse(xml).Root;
                                        foreach (var prop in elObject.Element("dmd").Element("properties").Elements("property"))
                                        {
                                            if (prop.Element("label") == null)
                                            {
                                                continue;
                                            }
                                            string label = prop.Element("label").Value.Replace(":", "");
                                            int order    = 1;
                                            foreach (var value in prop.Elements("valueGrp").Elements("value"))
                                            {
                                                tag = await TagHandler.PrepareAttribute(context, label, value.Value, order);
                                                if (value.Attribute("href") != null)
                                                {
                                                    if (value.Attribute("href").Value.IndexOf("http://localhost") != 0)
                                                    {
                                                        tag.ValueSupplement = value.Attribute("href").Value;
                                                    }
                                                }
                                                meta.Add(tag);

                                                if (label == "Title")
                                                {
                                                    book.Name            = book.NameInEnglish =
                                                        book.Description = book.DescriptionInEnglish =
                                                            value.Value;
                                                }
                                                order++;
                                            }
                                        }

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "Princeton Digital Library of Islamic Manuscripts", 1);
                                        tag.ValueSupplement = $"http://pudl.princeton.edu/objects/{job.ResourceNumber}";

                                        meta.Add(tag);


                                        book.Tags = meta.ToArray();
                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        foreach (var structure in elObject.Elements("structure"))
                                        {
                                            if (structure.Attribute("type") != null && structure.Attribute("type").Value == "RelatedObjects")
                                            {
                                                if (structure.Element("div") == null || structure.Element("div").Element("OrderedList") == null)
                                                {
                                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                    {
                                                        job.EndTime   = DateTime.Now;
                                                        job.Status    = ImportJobStatus.Failed;
                                                        job.Exception = "structure[RelatedObjects].div.OrderedList is null";
                                                        importJobUpdaterDb.Update(job);
                                                        await importJobUpdaterDb.SaveChangesAsync();
                                                        return;
                                                    }
                                                }


                                                int pageCount   = structure.Element("div").Element("OrderedList").Elements("div").Count();
                                                int inlineOrder = 0;

                                                foreach (var div in structure.Element("div").Element("OrderedList").Elements("div"))
                                                {
                                                    inlineOrder++;
                                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                    {
                                                        job.ProgressPercent = inlineOrder * 100 / (decimal)pageCount;
                                                        importJobUpdaterDb.Update(job);
                                                        await importJobUpdaterDb.SaveChangesAsync();
                                                    }

                                                    int order = int.Parse(div.Attribute("order").Value);
                                                    RArtifactItemRecord page = new RArtifactItemRecord()
                                                    {
                                                        Name                 = $"تصویر {order}",
                                                        NameInEnglish        = div.Attribute("label").Value,
                                                        Description          = "",
                                                        DescriptionInEnglish = "",
                                                        Order                = order,
                                                        FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                        LastModified         = DateTime.Now
                                                    };

                                                    string imageUrl = div.Attribute("img").Value;
                                                    imageUrl        = "https://libimages.princeton.edu/loris/" + imageUrl.Substring(imageUrl.LastIndexOf(":") + 1);
                                                    imageUrl       += $"/full/,{div.Attribute("h").Value}/0/default.jpg";

                                                    tag = await TagHandler.PrepareAttribute(context, "Source", "Princeton Digital Library of Islamic Manuscripts", 1);
                                                    tag.ValueSupplement = imageUrl;
                                                    page.Tags           = new RTagValue[] { tag };

                                                    if (!string.IsNullOrEmpty(imageUrl))
                                                    {
                                                        bool recovered = false;
                                                        if (
                                                            File.Exists
                                                            (
                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                            )
                                                            &&
                                                            File.Exists
                                                            (
                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                            )
                                                            &&
                                                            File.Exists
                                                            (
                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                            )
                                                            )
                                                        {
                                                            RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                                imageUrl,
                                                                                                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                                Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                                $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                            if (picture.Result != null)
                                                            {
                                                                recovered            = true;
                                                                page.Images          = new RPictureFile[] { picture.Result };
                                                                page.CoverImageIndex = 0;

                                                                if (book.CoverItemIndex == (order - 1))
                                                                {
                                                                    book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                                }

                                                                pages.Add(page);
                                                            }
                                                        }
                                                        if (!recovered)
                                                        {
                                                            if (
                                                                File.Exists
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                )

                                                                )
                                                            {
                                                                File.Delete
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                );
                                                            }
                                                            if (

                                                                File.Exists
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                )

                                                                )
                                                            {
                                                                File.Delete
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                );
                                                            }
                                                            if (

                                                                File.Exists
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                )
                                                                )
                                                            {
                                                                File.Delete
                                                                (
                                                                    Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                                );
                                                            }
                                                            var imageResult = await client.GetAsync(imageUrl);


                                                            int _ImportRetryCount        = 5;
                                                            int _ImportRetryInitialSleep = 500;
                                                            int retryCount = 0;
                                                            while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                            {
                                                                imageResult.Dispose();
                                                                Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                                imageResult = await client.GetAsync(imageUrl);
                                                                retryCount++;
                                                            }

                                                            if (imageResult.IsSuccessStatusCode)
                                                            {
                                                                using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                                {
                                                                    RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                                    if (picture.Result == null)
                                                                    {
                                                                        throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                                    }

                                                                    page.Images          = new RPictureFile[] { picture.Result };
                                                                    page.CoverImageIndex = 0;

                                                                    if (book.CoverItemIndex == (order - 1))
                                                                    {
                                                                        book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                                    }

                                                                    pages.Add(page);
                                                                }
                                                            }
                                                            else
                                                            {
                                                                using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                                {
                                                                    job.EndTime   = DateTime.Now;
                                                                    job.Status    = ImportJobStatus.Failed;
                                                                    job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                                    importJobUpdaterDb.Update(job);
                                                                    await importJobUpdaterDb.SaveChangesAsync();
                                                                }

                                                                imageResult.Dispose();
                                                                return;
                                                            }

                                                            imageResult.Dispose();
                                                            GC.Collect();
                                                        }
                                                    }
                                                }
                                            }
                                        }

                                        foreach (var structure in elObject.Elements("structure"))
                                        {
                                            if (structure.Attribute("type") != null && structure.Attribute("type").Value == "Physical")
                                            {
                                                if (structure.Element("RTLBoundManuscript") != null)
                                                {
                                                    foreach (var leaf in structure.Element("RTLBoundManuscript").Elements("Leaf"))
                                                    {
                                                        foreach (var side in leaf.Elements("Side"))
                                                        {
                                                            int pageOrder            = int.Parse(side.Attribute("order").Value);
                                                            tag                      = await TagHandler.PrepareAttribute(context, "Leaf Side", side.Attribute("label").Value, 100);
                                                            RArtifactItemRecord page = pages.Where(p => p.Order == pageOrder).SingleOrDefault();
                                                            if (page != null)
                                                            {
                                                                List <RTagValue> tags = new List <RTagValue>(page.Tags);
                                                                tags.Add(tag);
                                                                page.Tags = tags;
                                                            }
                                                        }
                                                    }
                                                    foreach (var folio in structure.Element("RTLBoundManuscript").Elements("Folio"))
                                                    {
                                                        foreach (var side in folio.Elements("Side"))
                                                        {
                                                            int pageOrder            = int.Parse(side.Attribute("order").Value);
                                                            tag                      = await TagHandler.PrepareAttribute(context, "Folio Side", folio.Attribute("label").Value + ":" + side.Attribute("label").Value, 101);
                                                            RArtifactItemRecord page = pages.Where(p => p.Order == pageOrder).SingleOrDefault();
                                                            if (page != null)
                                                            {
                                                                List <RTagValue> tags = new List <RTagValue>(page.Tags);
                                                                tags.Add(tag);
                                                                page.Tags = tags;
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }

                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (pages.Count == 0)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "ages.Count == 0";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );



                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
Example #11
0
        /// <summary>
        /// from http://www.library.upenn.edu/
        /// </summary>
        /// <param name="resourceNumber">MEDREN_9949222153503681</param>
        /// <param name="friendlyUrl"></param>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromPenLibraries(string resourceNumber, string friendlyUrl)
        {
            string url = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&rotation=0&size=0";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.PennLibraries && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated artifact friendly url '{friendlyUrl}'"));
            }



            ImportJob job = new ImportJob()
            {
                JobType         = JobType.PennLibraries,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();


            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (var client = new HttpClient())
                        {
                            client.Timeout = TimeSpan.FromMinutes(5);
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    using (RMuseumDbContext context = new RMuseumDbContext(Configuration))
                                    {
                                        if (
                                            (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync())
                                            !=
                                            null
                                            )
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "aborted because of duplicated friendly url";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = 0,
                                            FriendlyUrl    = friendlyUrl
                                        };


                                        List <RTagValue> meta = new List <RTagValue>();
                                        RTagValue tag;

                                        string html = await result.Content.ReadAsStringAsync();


                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                        {
                                            job.StartTime  = DateTime.Now;
                                            job.Status     = ImportJobStatus.Running;
                                            job.SrcContent = html;
                                            importJobUpdaterDb.Update(job);
                                            await importJobUpdaterDb.SaveChangesAsync();
                                        }

                                        string title  = "";
                                        string author = "";
                                        int tagOrder  = 1;

                                        int nIdxStart = html.IndexOf("https://repo.library.upenn.edu/djatoka/resolver?");
                                        if (nIdxStart == -1)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "https://repo.library.upenn.edu/djatoka/resolver? not found";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        string firstImageUrl = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart).Replace("&amp;", "&");

                                        nIdxStart = html.IndexOf("recordinfolabel");
                                        while (nIdxStart != -1)
                                        {
                                            nIdxStart             += "recordinfolabel\">".Length;
                                            int nIdxEnd            = html.IndexOf(":", nIdxStart);
                                            string recordinfolabel = html.Substring(nIdxStart, nIdxEnd - nIdxStart);
                                            nIdxStart              = html.IndexOf("recordinfotext", nIdxEnd);
                                            nIdxStart             += "recordinfotext\">".Length;
                                            nIdxEnd = html.IndexOf("</td>", nIdxStart);
                                            string recordinfotext = html.Substring(nIdxStart, nIdxEnd - nIdxStart).Replace("</div>", "<div>").Replace("\n", "").Replace("\r", "").Trim();

                                            string[] values = recordinfotext.Split("<div>", StringSplitOptions.RemoveEmptyEntries);

                                            foreach (string value in values)
                                            {
                                                if (value.Trim().Length == 0)
                                                {
                                                    continue;
                                                }
                                                if (recordinfolabel == "Title")
                                                {
                                                    title = value.Trim();
                                                    tag   = await TagHandler.PrepareAttribute(context, "Title", title, 1);
                                                    meta.Add(tag);
                                                }
                                                else
                                                if (recordinfolabel == "Author")
                                                {
                                                    author = value.Trim();
                                                    tag    = await TagHandler.PrepareAttribute(context, "Contributor Names", author, 1);
                                                    meta.Add(tag);
                                                }
                                                else
                                                {
                                                    tag = await TagHandler.PrepareAttribute(context, recordinfolabel, value.Trim(), tagOrder++);
                                                    meta.Add(tag);
                                                }
                                            }

                                            nIdxStart = html.IndexOf("recordinfolabel", nIdxEnd);
                                        }



                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "Penn Libraries", 1);
                                        string viewerUrl    = $"http://dla.library.upenn.edu/dla/medren/detail.html?id={resourceNumber}";
                                        tag.ValueSupplement = viewerUrl;

                                        meta.Add(tag);

                                        book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = title;
                                        book.Tags = meta.ToArray();

                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        int order = 0;
                                        while (true)
                                        {
                                            order++;

                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.ProgressPercent = order;
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }

                                            string imageUrl = firstImageUrl;

                                            RArtifactItemRecord page = new RArtifactItemRecord()
                                            {
                                                Name                 = $"تصویر {order}",
                                                NameInEnglish        = $"Image {order}",
                                                Description          = "",
                                                DescriptionInEnglish = "",
                                                Order                = order,
                                                FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                LastModified         = DateTime.Now
                                            };

                                            tag = await TagHandler.PrepareAttribute(context, "Source", "Penn Libraries", 1);
                                            tag.ValueSupplement = viewerUrl;
                                            page.Tags           = new RTagValue[] { tag };

                                            if (!string.IsNullOrEmpty(imageUrl))
                                            {
                                                bool recovered = false;
                                                if (
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    )
                                                {
                                                    RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                        imageUrl,
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                    if (picture.Result != null)
                                                    {
                                                        recovered            = true;
                                                        page.Images          = new RPictureFile[] { picture.Result };
                                                        page.CoverImageIndex = 0;

                                                        if (book.CoverItemIndex == (order - 1))
                                                        {
                                                            book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                        }

                                                        pages.Add(page);
                                                    }
                                                }
                                                if (!recovered)
                                                {
                                                    if (
                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )
                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }

                                                    if (order > 1)
                                                    {
                                                        string pageUrl = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&doubleside=0&rotation=0&size=0&currentpage={order}";
                                                        var pageResult = await client.GetAsync(pageUrl);

                                                        if (pageResult.StatusCode == HttpStatusCode.NotFound)
                                                        {
                                                            break;        //finished
                                                        }

                                                        string pageHtml = await pageResult.Content.ReadAsStringAsync();
                                                        nIdxStart       = pageHtml.IndexOf("https://repo.library.upenn.edu/djatoka/resolver?");
                                                        if (nIdxStart == -1)
                                                        {
                                                            if (order > 1)
                                                            {
                                                                break;         //finished
                                                            }
                                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                                            {
                                                                job.EndTime   = DateTime.Now;
                                                                job.Status    = ImportJobStatus.Failed;
                                                                job.Exception = $"https://repo.library.upenn.edu/djatoka/resolver? not found on page {order}";
                                                                importJobUpdaterDb.Update(job);
                                                                await importJobUpdaterDb.SaveChangesAsync();
                                                            }
                                                            return;
                                                        }

                                                        imageUrl = pageHtml.Substring(nIdxStart, pageHtml.IndexOf('"', nIdxStart) - nIdxStart).Replace("&amp;", "&");
                                                    }
                                                    var imageResult = await client.GetAsync(imageUrl);

                                                    if (imageResult.StatusCode == HttpStatusCode.NotFound)
                                                    {
                                                        break;        //finished
                                                    }


                                                    int _ImportRetryCount        = 200;
                                                    int _ImportRetryInitialSleep = 500;
                                                    int retryCount = 0;
                                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                    {
                                                        imageResult.Dispose();
                                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                        imageResult = await client.GetAsync(imageUrl);
                                                        retryCount++;
                                                    }

                                                    if (imageResult.IsSuccessStatusCode)
                                                    {
                                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                        {
                                                            RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                            if (picture.Result == null)
                                                            {
                                                                throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                            }


                                                            page.Images          = new RPictureFile[] { picture.Result };
                                                            page.CoverImageIndex = 0;

                                                            if (book.CoverItemIndex == (order - 1))
                                                            {
                                                                book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                            }

                                                            pages.Add(page);
                                                        }
                                                    }
                                                    else
                                                    {
                                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                                        {
                                                            job.EndTime   = DateTime.Now;
                                                            job.Status    = ImportJobStatus.Failed;
                                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                            importJobUpdaterDb.Update(job);
                                                            await importJobUpdaterDb.SaveChangesAsync();
                                                        }

                                                        imageResult.Dispose();
                                                        return;
                                                    }

                                                    imageResult.Dispose();
                                                    GC.Collect();
                                                }
                                            }
                                        }


                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (pages.Count == 0)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                            {
                                                job.EndTime   = DateTime.Now;
                                                job.Status    = ImportJobStatus.Failed;
                                                job.Exception = "Pages.Count == 0";
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }
                                            return;
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );



                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }
        /// <summary>
        /// from https://www.loc.gov
        /// </summary>
        /// <param name="resourceNumber">
        /// <example>
        /// m084
        /// </example>
        /// </param>
        /// <param name="friendlyUrl">
        /// <example>
        /// boostan1207
        /// </example>
        /// </param>
        /// <param name="resourcePrefix"></param>
        /// <example>
        /// plmp
        /// </example>
        /// <returns></returns>
        private async Task <RServiceResult <bool> > StartImportingFromTheLibraryOfCongress(string resourceNumber, string friendlyUrl, string resourcePrefix)
        {
            string url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?fo=json&st=gallery";

            if (
                (
                    await _context.ImportJobs
                    .Where(j => j.JobType == JobType.Loc && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted))
                    .SingleOrDefaultAsync()
                )
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"Job is already scheduled or running for importing {url}"));
            }

            if (string.IsNullOrEmpty(friendlyUrl))
            {
                friendlyUrl = resourceNumber;
            }

            if (
                (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync())
                !=
                null
                )
            {
                return(new RServiceResult <bool>(false, $"duplicated friendly url '{friendlyUrl}'"));
            }

            ImportJob job = new ImportJob()
            {
                JobType         = JobType.Loc,
                ResourceNumber  = resourceNumber,
                FriendlyUrl     = friendlyUrl,
                SrcUrl          = url,
                QueueTime       = DateTime.Now,
                ProgressPercent = 0,
                Status          = ImportJobStatus.NotStarted
            };


            await _context.ImportJobs.AddAsync
            (
                job
            );

            await _context.SaveChangesAsync();

            try
            {
                _backgroundTaskQueue.QueueBackgroundWorkItem
                (
                    async token =>
                {
                    try
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.StartTime = DateTime.Now;
                            job.Status    = ImportJobStatus.Running;
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }

                        int pageCount            = 0;
                        int representative_index = 0;
                        //اول یک صفحه را می‌خوانیم تا تعداد صفحات را مشخص کنیم
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    string json = await result.Content.ReadAsStringAsync();
                                    var parsed  = JObject.Parse(json);

                                    pageCount            = parsed.SelectToken("resource.segment_count").Value <int>();
                                    representative_index = parsed.SelectToken("resource.representative_index").Value <int>();
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }

                        //here might be problems: loc json does not return correct answer when number of segments are more than 1000

                        /*
                         * if (pageCount > 1000)
                         * {
                         *  job.Exception = $"Page count ({pageCount}) was cut to 1000 for this artifact due to loc bug.";
                         *  pageCount = 1000;
                         * }
                         */

                        //حالا که تعداد صفحات را داریم دوباره می‌خوانیم
                        url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?c={pageCount}&fo=json&st=gallery";
                        using (var client = new HttpClient())
                        {
                            using (var result = await client.GetAsync(url))
                            {
                                if (result.IsSuccessStatusCode)
                                {
                                    //here is a problem, this method could be called from a background service where _context is disposed, so I need to renew it
                                    using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}")
                                        {
                                            Status         = PublishStatus.Draft,
                                            DateTime       = DateTime.Now,
                                            LastModified   = DateTime.Now,
                                            CoverItemIndex = representative_index,
                                            FriendlyUrl    = friendlyUrl
                                        };

                                        string json = await result.Content.ReadAsStringAsync();

                                        job.SrcContent = json;

                                        var parsed = JObject.Parse(json);

                                        var segmentsArray = parsed.SelectToken("segments").ToArray();
                                        //here might be problems: loc json does not return correct answer when number of segments are more than 1000
                                        //I've added some temporary solutions prior
                                        //Here I want to log any paradox I encounter:
                                        if (segmentsArray.Length != pageCount)
                                        {
                                            job.Exception = $"Page count ({pageCount}) is not equal to number of returned resources ({segmentsArray.Length}).";
                                        }



                                        List <RTagValue> meta = new List <RTagValue>();

                                        string string_value = await HandleSimpleValue(context, parsed, meta, "item.title", "Title");
                                        if (!string.IsNullOrWhiteSpace(string_value))
                                        {
                                            book.Name          = string_value;
                                            book.NameInEnglish = string_value;
                                        }
                                        await HandleSimpleValue(context, parsed, meta, "item.date", "Date");
                                        string_value = await HandleListValue(context, parsed, meta, "item.other_title", "Other Title");
                                        if (!string.IsNullOrWhiteSpace(string_value))
                                        {
                                            book.Name = string_value;
                                        }
                                        await HandleListValue(context, parsed, meta, "item.contributor_names", "Contributor Names");
                                        await HandleSimpleValue(context, parsed, meta, "item.shelf_id", "Shelf ID");
                                        await HandleListValue(context, parsed, meta, "item.created_published", "Created / Published");
                                        await HandleListValue(context, parsed, meta, "item.subject_headings", "Subject Headings");
                                        await HandleListValue(context, parsed, meta, "item.notes", "Notes");
                                        await HandleListValue(context, parsed, meta, "item.medium", "Medium");
                                        await HandleListValue(context, parsed, meta, "item.call_number", "Call Number/Physical Location");
                                        await HandleListValue(context, parsed, meta, "item.digital_id", "Digital Id");
                                        await HandleSimpleValue(context, parsed, meta, "item.library_of_congress_control_number", "Library of Congress Control Number");
                                        await HandleChildrenValue(context, parsed, meta, "item.language", "Language");
                                        await HandleListValue(context, parsed, meta, "item.online_format", "Online Format");
                                        await HandleListValue(context, parsed, meta, "item.number_oclc", "OCLC Number");
                                        string_value = await HandleListValue(context, parsed, meta, "item.description", "Description");
                                        if (!string.IsNullOrEmpty(string_value))
                                        {
                                            book.Description          = string_value;
                                            book.DescriptionInEnglish = string_value;
                                        }
                                        await HandleSimpleValue(context, parsed, meta, "cite_this.chicago", "Chicago citation style");
                                        await HandleSimpleValue(context, parsed, meta, "cite_this.apa", "APA citation style");
                                        await HandleSimpleValue(context, parsed, meta, "cite_this.mla", "MLA citation style");
                                        await HandleChildrenValue(context, parsed, meta, "item.dates", "Dates");
                                        await HandleChildrenValue(context, parsed, meta, "item.contributors", "Contributors");
                                        await HandleChildrenValue(context, parsed, meta, "item.location", "Location");
                                        await HandleListValue(context, parsed, meta, "item.rights", "Rights & Access");

                                        RTagValue tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1);
                                        meta.Add(tag);

                                        tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1);
                                        meta.Add(tag);


                                        tag = await TagHandler.PrepareAttribute(context, "Source", "Library of Congress, African and Middle East Division, Near East Section Persian Manuscript Collection", 1);
                                        tag.ValueSupplement = url;
                                        string_value        = parsed.SelectToken("item.id").Value <string>();
                                        if (!string.IsNullOrWhiteSpace(string_value))
                                        {
                                            tag.ValueSupplement = string_value;
                                        }

                                        meta.Add(tag);


                                        book.Tags = meta.ToArray();



                                        int order = 0;
                                        List <RArtifactItemRecord> pages = new List <RArtifactItemRecord>();

                                        //due to loc bug for books with more than 1000 pages relying on segmentsArray changed to hard coded image urls and ....
                                        //foreach (JToken segment in segmentsArray)
                                        for (int pageIndex = 1; pageIndex <= pageCount; pageIndex++)
                                        {
                                            using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                            {
                                                job.ProgressPercent = order * 100 / (decimal)pageCount;
                                                importJobUpdaterDb.Update(job);
                                                await importJobUpdaterDb.SaveChangesAsync();
                                            }

                                            order++;



                                            RArtifactItemRecord page = new RArtifactItemRecord()
                                            {
                                                Name                 = $"تصویر {order}",
                                                NameInEnglish        = $"Image {pageIndex} of {book.NameInEnglish}", //segment.SelectToken("title").Value<string>(),
                                                Description          = "",
                                                DescriptionInEnglish = "",
                                                Order                = order,
                                                FriendlyUrl          = $"p{$"{order}".PadLeft(4, '0')}",
                                                LastModified         = DateTime.Now
                                            };

                                            tag = await TagHandler.PrepareAttribute(context, "Source", "Library of Congress, African and Middle East Division, Near East Section Persian Manuscript Collection", 1);
                                            tag.ValueSupplement = $"http://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?sp={pageIndex}";        //segment.SelectToken("id").Value<string>();
                                            page.Tags           = new RTagValue[] { tag };

                                            string imageUrlPart = $"{pageIndex}".PadLeft(4, '0');
                                            string imageUrl     = $"https://tile.loc.gov/image-services/iiif/service:amed:{resourcePrefix}:{resourceNumber}:{imageUrlPart}/full/pct:100/0/default.jpg";
                                            //string imageUrl = $"https://tile.loc.gov/image-services/iiif/service:rbc:{resourcePrefix}:2015:{resourceNumber}:{imageUrlPart}/full/pct:100/0/default.jpg";

                                            /*
                                             * List<string> list = segment.SelectToken("image_url").ToObject<List<string>>();
                                             * if (list != null && list.Count > 0)
                                             * {
                                             *  for (int i = 0; i < list.Count; i++)
                                             *  {
                                             *      if (list[i].IndexOf(".jpg") != -1)
                                             *      {
                                             *          if (imageUrl == "")
                                             *              imageUrl = list[i];
                                             *          else
                                             *          {
                                             *              if (imageUrl.IndexOf("#h=") != -1 && imageUrl.IndexOf("&w=", imageUrl.IndexOf("#h=")) != -1)
                                             *              {
                                             *                  int h1 = int.Parse(imageUrl.Substring(imageUrl.IndexOf("#h=") + "#h=".Length, imageUrl.IndexOf("&w=") - imageUrl.IndexOf("#h=") - "&w=".Length));
                                             *                  if (list[i].IndexOf("#h=") != -1 && list[i].IndexOf("&w=", list[i].IndexOf("#h=")) != -1)
                                             *                  {
                                             *                      int h2 = int.Parse(list[i].Substring(list[i].IndexOf("#h=") + "#h=".Length, list[i].IndexOf("&w=") - list[i].IndexOf("#h=") - "&w=".Length));
                                             *
                                             *                      if (h2 > h1)
                                             *                      {
                                             *                          imageUrl = list[i];
                                             *                      }
                                             *                  }
                                             *              }
                                             *              else
                                             *                  imageUrl = list[i];
                                             *
                                             *          }
                                             *      }
                                             *  }
                                             * }
                                             */



                                            if (!string.IsNullOrEmpty(imageUrl))
                                            {
                                                //imageUrl = "https:" + imageUrl.Substring(0, imageUrl.IndexOf('#'));
                                                bool recovered = false;
                                                if (
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    &&
                                                    File.Exists
                                                    (
                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                    )
                                                    )
                                                {
                                                    RServiceResult <RPictureFile> picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1,
                                                                                                                                        imageUrl,
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"),
                                                                                                                                        $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                    if (picture.Result != null)
                                                    {
                                                        recovered            = true;
                                                        page.Images          = new RPictureFile[] { picture.Result };
                                                        page.CoverImageIndex = 0;

                                                        if (book.CoverItemIndex == (order - 1))
                                                        {
                                                            book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                        }

                                                        pages.Add(page);
                                                    }
                                                }
                                                if (!recovered)
                                                {
                                                    if (
                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )

                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    if (

                                                        File.Exists
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        )
                                                        )
                                                    {
                                                        File.Delete
                                                        (
                                                            Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg")
                                                        );
                                                    }
                                                    var imageResult = await client.GetAsync(imageUrl);


                                                    int _ImportRetryCount        = 5;
                                                    int _ImportRetryInitialSleep = 500;
                                                    int retryCount = 0;
                                                    while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable)
                                                    {
                                                        imageResult.Dispose();
                                                        Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1));
                                                        imageResult = await client.GetAsync(imageUrl);
                                                        retryCount++;
                                                    }

                                                    if (imageResult.IsSuccessStatusCode)
                                                    {
                                                        using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
                                                        {
                                                            RServiceResult <RPictureFile> picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl);
                                                            if (picture.Result == null)
                                                            {
                                                                throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}");
                                                            }

                                                            page.Images          = new RPictureFile[] { picture.Result };
                                                            page.CoverImageIndex = 0;

                                                            if (book.CoverItemIndex == (order - 1))
                                                            {
                                                                book.CoverImage = RPictureFile.Duplicate(picture.Result);
                                                            }

                                                            pages.Add(page);
                                                        }
                                                    }
                                                    else
                                                    {
                                                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                                        {
                                                            job.EndTime   = DateTime.Now;
                                                            job.Status    = ImportJobStatus.Failed;
                                                            job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}";
                                                            importJobUpdaterDb.Update(job);
                                                            await importJobUpdaterDb.SaveChangesAsync();
                                                        }

                                                        imageResult.Dispose();
                                                        return;
                                                    }

                                                    imageResult.Dispose();
                                                    GC.Collect();
                                                }
                                            }
                                        }

                                        book.Items     = pages.ToArray();
                                        book.ItemCount = pages.Count;

                                        if (book.CoverImage == null && pages.Count > 0)
                                        {
                                            book.CoverImage = RPictureFile.Duplicate(pages[0].Images.First());
                                        }

                                        await context.Artifacts.AddAsync(book);
                                        await context.SaveChangesAsync();

                                        job.ProgressPercent = 100;
                                        job.Status          = ImportJobStatus.Succeeded;
                                        job.ArtifactId      = book.Id;
                                        job.EndTime         = DateTime.Now;
                                        context.Update(job);
                                        await context.SaveChangesAsync();
                                    }
                                }
                                else
                                {
                                    using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                                    {
                                        job.EndTime   = DateTime.Now;
                                        job.Status    = ImportJobStatus.Failed;
                                        job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}";
                                        importJobUpdaterDb.Update(job);
                                        await importJobUpdaterDb.SaveChangesAsync();
                                    }
                                    return;
                                }
                            }
                        }
                    }
                    catch (Exception exp)
                    {
                        using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>()))
                        {
                            job.EndTime   = DateTime.Now;
                            job.Status    = ImportJobStatus.Failed;
                            job.Exception = exp.ToString();
                            importJobUpdaterDb.Update(job);
                            await importJobUpdaterDb.SaveChangesAsync();
                        }
                    }
                }
                );



                return(new RServiceResult <bool>(true));
            }
            catch (Exception exp)
            {
                return(new RServiceResult <bool>(false, exp.ToString()));
            }
        }