/// <summary>
/// Finds the image in the local folder (using EAN and size) and uploads to ipfs
/// </summary>
/// <param name="localImageCacheFolder"></param>
/// <param name="ean"></param>
/// <param name="imageSize"></param>
/// <param name="imageSourceUrl"></param>
/// <param name="ipfsService"></param>
/// <param name="retryNumber"></param>
/// <returns></returns>
private static async Task <IpfsBookCoverMapping> UploadCoverToIpfs(
string localImageCacheFolder,
string ean,
string imageSize,
string imageSourceUrl,
IpfsService ipfsService,
int retryNumber = 0)
{
string locallyCachedCopy = CreateLocalEanFilePath(ean, imageSize, imageSourceUrl, localImageCacheFolder);
if (!File.Exists(locallyCachedCopy))
{
return(null);
}
ipfsFilesLoaded = Interlocked.Increment(ref ipfsFilesLoaded);
Console.WriteLine($"uploading file to ipfs: {ean} {imageSize} ({ipfsFilesLoaded})");
try
{
var node = await ipfsService.AddFileAsync(locallyCachedCopy);
return(new IpfsBookCoverMapping
{
Ean = ean,
ImageSourceUrl = imageSourceUrl,
IpfsHash = node.Hash.ToString(),
IpfsUrl = ipfsService.GetUrl(node.Hash.ToString()),
ImageSize = imageSize,
EanUrl = Path.GetFileName(locallyCachedCopy),
FullEanUrl = locallyCachedCopy
});
}
catch
{
retryNumber++;
if (retryNumber == 3)
{
throw;
}
await Task.Delay(5000 *retryNumber);
return(await UploadCoverToIpfs(localImageCacheFolder, ean, imageSize, imageSourceUrl, ipfsService, retryNumber));
}
}
/// <summary>
/// Uploads book cover images to ipfs
/// Writes a mapping csv file to relate ean, image size, ipfs hash etc
/// Assumes the book cover images are held locally so....
/// Ensure the cover images have been downloaded first! (DownloadCoversAsync)
/// </summary>
/// <param name="books"></param>
/// <returns></returns>
public static async Task UploadCoversToIpfsAsync(
IEnumerable <Book> books, string coverFolder, string indexOutputFilePath, string ipfsUrl)
{
ipfsFilesLoaded = 0;
//indexOutputFilePath
try
{
// we'll parallel load images - but don't want to write to the csv in parallel
var semaphore = new SemaphoreSlim(1);
using (var textWriter = File.CreateText(indexOutputFilePath))
{
var csvWriter = new CsvHelper.CsvWriter(textWriter, CultureInfo.InvariantCulture);
csvWriter.WriteHeader <IpfsBookCoverMapping>();
csvWriter.NextRecord();
try
{
var ipfsService = new IpfsService(ipfsUrl);
foreach (var book in books.AsParallel())
{
if (!string.IsNullOrWhiteSpace(book.COVER_SMALL))
{
var smallCover = await UploadCoverToIpfs(coverFolder, book.EAN, "s", book.COVER_SMALL, ipfsService);
await Save(smallCover);
}
if (!string.IsNullOrWhiteSpace(book.COVER_MED))
{
var mediumCover = await UploadCoverToIpfs(coverFolder, book.EAN, "m", book.COVER_MED, ipfsService);
await Save(mediumCover);
}
if (!string.IsNullOrWhiteSpace(book.COVER_LARGE))
{
var largeCover = await UploadCoverToIpfs(coverFolder, book.EAN, "l", book.COVER_LARGE, ipfsService);
await Save(largeCover);
}
async Task Save(IpfsBookCoverMapping mapping)
{
await semaphore.WaitAsync();
try
{
csvWriter.WriteRecord(mapping);
csvWriter.NextRecord();
}
finally
{
semaphore.Release();
}
}
}
}
finally
{
await csvWriter.FlushAsync();
await textWriter.FlushAsync();
}
}
}
finally
{
// consider changing file name on completion to avoid it being overwritten accidentally
}
}