/// <summary> /// Finds the image in the local folder (using EAN and size) and uploads to ipfs /// </summary> /// <param name="localImageCacheFolder"></param> /// <param name="ean"></param> /// <param name="imageSize"></param> /// <param name="imageSourceUrl"></param> /// <param name="ipfsService"></param> /// <param name="retryNumber"></param> /// <returns></returns> private static async Task <IpfsBookCoverMapping> UploadCoverToIpfs( string localImageCacheFolder, string ean, string imageSize, string imageSourceUrl, IpfsService ipfsService, int retryNumber = 0) { string locallyCachedCopy = CreateLocalEanFilePath(ean, imageSize, imageSourceUrl, localImageCacheFolder); if (!File.Exists(locallyCachedCopy)) { return(null); } ipfsFilesLoaded = Interlocked.Increment(ref ipfsFilesLoaded); Console.WriteLine($"uploading file to ipfs: {ean} {imageSize} ({ipfsFilesLoaded})"); try { var node = await ipfsService.AddFileAsync(locallyCachedCopy); return(new IpfsBookCoverMapping { Ean = ean, ImageSourceUrl = imageSourceUrl, IpfsHash = node.Hash.ToString(), IpfsUrl = ipfsService.GetUrl(node.Hash.ToString()), ImageSize = imageSize, EanUrl = Path.GetFileName(locallyCachedCopy), FullEanUrl = locallyCachedCopy }); } catch { retryNumber++; if (retryNumber == 3) { throw; } await Task.Delay(5000 *retryNumber); return(await UploadCoverToIpfs(localImageCacheFolder, ean, imageSize, imageSourceUrl, ipfsService, retryNumber)); } }
/// <summary> /// Uploads book cover images to ipfs /// Writes a mapping csv file to relate ean, image size, ipfs hash etc /// Assumes the book cover images are held locally so.... /// Ensure the cover images have been downloaded first! (DownloadCoversAsync) /// </summary> /// <param name="books"></param> /// <returns></returns> public static async Task UploadCoversToIpfsAsync( IEnumerable <Book> books, string coverFolder, string indexOutputFilePath, string ipfsUrl) { ipfsFilesLoaded = 0; //indexOutputFilePath try { // we'll parallel load images - but don't want to write to the csv in parallel var semaphore = new SemaphoreSlim(1); using (var textWriter = File.CreateText(indexOutputFilePath)) { var csvWriter = new CsvHelper.CsvWriter(textWriter, CultureInfo.InvariantCulture); csvWriter.WriteHeader <IpfsBookCoverMapping>(); csvWriter.NextRecord(); try { var ipfsService = new IpfsService(ipfsUrl); foreach (var book in books.AsParallel()) { if (!string.IsNullOrWhiteSpace(book.COVER_SMALL)) { var smallCover = await UploadCoverToIpfs(coverFolder, book.EAN, "s", book.COVER_SMALL, ipfsService); await Save(smallCover); } if (!string.IsNullOrWhiteSpace(book.COVER_MED)) { var mediumCover = await UploadCoverToIpfs(coverFolder, book.EAN, "m", book.COVER_MED, ipfsService); await Save(mediumCover); } if (!string.IsNullOrWhiteSpace(book.COVER_LARGE)) { var largeCover = await UploadCoverToIpfs(coverFolder, book.EAN, "l", book.COVER_LARGE, ipfsService); await Save(largeCover); } async Task Save(IpfsBookCoverMapping mapping) { await semaphore.WaitAsync(); try { csvWriter.WriteRecord(mapping); csvWriter.NextRecord(); } finally { semaphore.Release(); } } } } finally { await csvWriter.FlushAsync(); await textWriter.FlushAsync(); } } } finally { // consider changing file name on completion to avoid it being overwritten accidentally } }