public static async Task Run(string path, long startId, bool ignoreHashCheck, bool includeDeleted) { string tempFolderPath = Path.Combine(path, "_temp"); string imageFolderPath = Path.Combine(path, "images"); string metadataDatabasePath = Path.Combine(path, "danbooru.sqlite"); string lastPostJsonPath = Path.Combine(path, "last_post.json"); PathUtility.CreateDirectoryIfNotExists(path); PathUtility.CreateDirectoryIfNotExists(tempFolderPath); PathUtility.CreateDirectoryIfNotExists(imageFolderPath); using (SqliteConnection connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = metadataDatabasePath, }.ToString())) { connection.Open(); SQLiteUtility.TryCreateTable(connection); while (true) { // Get posts metadata as json JObject[] postJObjects = null; await TaskUtility.RunWithRetry(async() => { Log.Info($"Downloading metadata ... ({startId} ~ )"); postJObjects = await DanbooruUtility.GetPosts(startId); }, e => { Log.Error(e); return(true); }, 10, 3000); if (postJObjects.Length == 0) { Log.Info("There is no posts."); break; } // Validate post Log.Info($"Checking {postJObjects.Length} posts ..."); Post[] posts = postJObjects.Select(p => ConvertToPost(p)).Where(p => p != null).ToArray(); Parallel.ForEach(posts, post => { if (string.IsNullOrEmpty(post.Md5)) { Log.Debug($"Skip for empty MD5 : Id={post.Id}"); return; } if (string.IsNullOrEmpty(post.ImageUrl)) { Log.Debug($"Skip for empty image URL : Id={post.Id}"); return; } if (post.IsDeleted && !includeDeleted) { return; } if (post.IsPending) { return; } post.IsValid = true; string metadataPath = GetPostLocalMetadataPath(imageFolderPath, post); try { if (File.Exists(metadataPath)) { Post cachedPost = ConvertToPost(JObject.Parse(File.ReadAllText(metadataPath))); if (cachedPost == null || post.UpdatedDate > cachedPost.UpdatedDate) { post.ShouldSaveMetadata = true; post.ShouldUpdateImage = true; } } else { post.ShouldSaveMetadata = true; post.ShouldUpdateImage = true; } } catch (Exception e) { post.ShouldSaveMetadata = true; Log.Error(e); } string imagePath = GetPostLocalImagePath(imageFolderPath, post); if (!File.Exists(imagePath)) { post.ShouldDownloadImage = true; return; } else { if (post.ShouldUpdateImage || !ignoreHashCheck) { string cachedImageMd5 = GetMd5Hash(imagePath); if (post.Md5 != cachedImageMd5) { post.ShouldDownloadImage = true; Log.Info($"MD5 is different to cached image : Id={post.Id}, {post.Md5} (new) != {cachedImageMd5} (cached)"); return; } } } }); int shouldDownloadCount = posts.Where(p => p.ShouldDownloadImage).Count(); int shouldUpdateCount = posts.Where(p => p.ShouldSaveMetadata).Count(); int pendingCount = posts.Where(p => p.IsPending).Count(); if (shouldUpdateCount > 0 || shouldDownloadCount > 0) { Log.Info($"{shouldUpdateCount}/{posts.Length} posts are updated. {pendingCount} posts are pending. Downloading {shouldDownloadCount} posts ..."); } foreach (Post post in posts) { if (!post.IsValid) { continue; } string metadataPath = GetPostLocalMetadataPath(imageFolderPath, post); string imagePath = GetPostLocalImagePath(imageFolderPath, post); string tempImagePath = GetPostTempImagePath(tempFolderPath, post); PathUtility.CreateDirectoryIfNotExists(Path.GetDirectoryName(imagePath)); try { await TaskUtility.RunWithRetry(async() => { if (post.ShouldDownloadImage) { Log.Info($"Downloading post {post.Id} ..."); await Download(post.ImageUrl, tempImagePath); string downloadedMd5 = GetMd5Hash(tempImagePath); if (downloadedMd5 != post.Md5) { Log.Warn($"MD5 hash of downloaded image is different : Id={post.Id}, {post.Md5} (metadata) != {downloadedMd5} (downloaded)"); try { File.Delete(tempImagePath); } finally { } try { File.Delete(metadataPath); } finally { } throw new Exception(); } File.Delete(imagePath); File.Move(tempImagePath, imagePath); } if (post.ShouldDownloadImage || post.ShouldSaveMetadata) { PathUtility.ChangeFileTimestamp(imagePath, post.CreatedDate, post.UpdatedDate); } if (post.ShouldSaveMetadata) { File.WriteAllText(metadataPath, post.JObject.ToString()); } }, e => { return(!(e is NotRetryableException)); }, 10, 3000); } catch (NotRetryableException) { Log.Error($"Can't retryable exception was occured : Id={post.Id}"); post.IsValid = false; } } Log.Info("Updating database ..."); SQLiteUtility.InsertOrReplace(connection, posts.Where(p => p.IsValid).Select(p => p.JObject)); long lastId = long.Parse(posts.Last().Id); startId = lastId + 1; } try { Directory.Delete(tempFolderPath, true); } catch (Exception e) { Log.Warn(e); } Log.Info("Dump command is complete."); } }