public List <ImportResult> Import(List <ImportDecision <LocalBook> > decisions, bool replaceExisting, DownloadClientItem downloadClientItem = null, ImportMode importMode = ImportMode.Auto) { var importResults = new List <ImportResult>(); var allImportedTrackFiles = new List <BookFile>(); var allOldTrackFiles = new List <BookFile>(); var addedAuthors = new List <Author>(); var bookDecisions = decisions.Where(e => e.Item.Book != null && e.Approved) .GroupBy(e => e.Item.Book.ForeignBookId).ToList(); var iDecision = 1; foreach (var albumDecision in bookDecisions) { _logger.ProgressInfo($"Importing book {iDecision++}/{bookDecisions.Count} {albumDecision.First().Item.Book}"); var decisionList = albumDecision.ToList(); var author = EnsureAuthorAdded(decisionList, addedAuthors); if (author == null) { // failed to add the author, carry on with next book continue; } var book = EnsureBookAdded(decisionList); if (book == null) { // failed to add the book, carry on with next one continue; } if (replaceExisting) { RemoveExistingTrackFiles(author, book); } // set the correct release to be monitored before importing the new files var newRelease = albumDecision.First().Item.Edition; _logger.Debug("Updating release to {0}", newRelease); book.Editions = _editionService.SetMonitored(newRelease); // Publish book edited event. // Deliberatly don't put in the old book since we don't want to trigger an ArtistScan. _eventAggregator.PublishEvent(new BookEditedEvent(book, book)); } var qualifiedImports = decisions.Where(c => c.Approved) .GroupBy(c => c.Item.Author.Id, (i, s) => s .OrderByDescending(c => c.Item.Quality, new QualityModelComparer(s.First().Item.Author.QualityProfile)) .ThenByDescending(c => c.Item.Size)) .SelectMany(c => c) .ToList(); _logger.ProgressInfo($"Importing {qualifiedImports.Count} files"); _logger.Debug($"Importing {qualifiedImports.Count} files. replaceExisting: {replaceExisting}"); var filesToAdd = new List <BookFile>(qualifiedImports.Count); var trackImportedEvents = new List <TrackImportedEvent>(qualifiedImports.Count); foreach (var importDecision in qualifiedImports.OrderByDescending(e => e.Item.Size)) { var localTrack = importDecision.Item; var oldFiles = new List <BookFile>(); try { //check if already imported if (importResults.Select(r => r.ImportDecision.Item.Book.Id).Contains(localTrack.Book.Id)) { if (!(localTrack.FileTrackInfo.TrackNumbers != null && localTrack.FileTrackInfo.TrackNumbers.Any())) { importResults.Add(new ImportResult(importDecision, "Book has already been imported")); continue; } else { var matchingImportResults = importResults.Where(r => r.ImportDecision.Item.Book.Id == localTrack.Book.Id); if (matchingImportResults.Select(r => r.ImportDecision.Item.FileTrackInfo.TrackNumbers).Contains(localTrack.FileTrackInfo.TrackNumbers)) { importResults.Add(new ImportResult(importDecision, "Audiobook track has already been imported")); continue; } } } localTrack.Book.Author = localTrack.Author; var bookFile = new BookFile { Path = localTrack.Path.CleanFilePath(), Size = localTrack.Size, Modified = localTrack.Modified, DateAdded = DateTime.UtcNow, ReleaseGroup = localTrack.ReleaseGroup, Quality = localTrack.Quality, MediaInfo = localTrack.FileTrackInfo.MediaInfo, EditionId = localTrack.Edition.Id, Author = localTrack.Author, Edition = localTrack.Edition }; bool copyOnly; switch (importMode) { default: case ImportMode.Auto: copyOnly = downloadClientItem != null && !downloadClientItem.CanMoveFiles; break; case ImportMode.Move: copyOnly = false; break; case ImportMode.Copy: copyOnly = true; break; } if (!localTrack.ExistingFile) { bookFile.SceneName = GetSceneReleaseName(downloadClientItem); var moveResult = _bookFileUpgrader.UpgradeBookFile(bookFile, localTrack, copyOnly); oldFiles = moveResult.OldFiles; } else { // Delete existing files from the DB mapped to this path var previousFile = _mediaFileService.GetFileWithPath(bookFile.Path); if (previousFile != null) { _mediaFileService.Delete(previousFile, DeleteMediaFileReason.ManualOverride); } var rootFolder = _rootFolderService.GetBestRootFolder(localTrack.Path); if (rootFolder.IsCalibreLibrary) { bookFile.CalibreId = bookFile.Path.ParseCalibreId(); } _audioTagService.WriteTags(bookFile, false); } filesToAdd.Add(bookFile); importResults.Add(new ImportResult(importDecision)); if (!localTrack.ExistingFile) { _extraService.ImportTrack(localTrack, bookFile, copyOnly); } allImportedTrackFiles.Add(bookFile); allOldTrackFiles.AddRange(oldFiles); // create all the import events here, but we can't publish until the trackfiles have been // inserted and ids created trackImportedEvents.Add(new TrackImportedEvent(localTrack, bookFile, oldFiles, !localTrack.ExistingFile, downloadClientItem)); } catch (RootFolderNotFoundException e) { _logger.Warn(e, "Couldn't import book " + localTrack); _eventAggregator.PublishEvent(new TrackImportFailedEvent(e, localTrack, !localTrack.ExistingFile, downloadClientItem)); importResults.Add(new ImportResult(importDecision, "Failed to import book, Root folder missing.")); } catch (DestinationAlreadyExistsException e) { _logger.Warn(e, "Couldn't import book " + localTrack); importResults.Add(new ImportResult(importDecision, "Failed to import book, Destination already exists.")); } catch (UnauthorizedAccessException e) { _logger.Warn(e, "Couldn't import book " + localTrack); _eventAggregator.PublishEvent(new TrackImportFailedEvent(e, localTrack, !localTrack.ExistingFile, downloadClientItem)); importResults.Add(new ImportResult(importDecision, "Failed to import book, Permissions error")); } catch (Exception) { throw; } } var watch = new System.Diagnostics.Stopwatch(); watch.Start(); _mediaFileService.AddMany(filesToAdd); _logger.Debug($"Inserted new trackfiles in {watch.ElapsedMilliseconds}ms"); // now that trackfiles have been inserted and ids generated, publish the import events foreach (var trackImportedEvent in trackImportedEvents) { _eventAggregator.PublishEvent(trackImportedEvent); } var albumImports = importResults.Where(e => e.ImportDecision.Item.Book != null) .GroupBy(e => e.ImportDecision.Item.Book.Id).ToList(); foreach (var albumImport in albumImports) { var book = albumImport.First().ImportDecision.Item.Book; var author = albumImport.First().ImportDecision.Item.Author; if (albumImport.Where(e => e.Errors.Count == 0).ToList().Count > 0 && author != null && book != null) { _eventAggregator.PublishEvent(new BookImportedEvent( author, book, allImportedTrackFiles.Where(s => s.EditionId == book.Id).ToList(), allOldTrackFiles.Where(s => s.EditionId == book.Id).ToList(), replaceExisting, downloadClientItem)); } } //Adding all the rejected decisions importResults.AddRange(decisions.Where(c => !c.Approved) .Select(d => new ImportResult(d, d.Rejections.Select(r => r.Reason).ToArray()))); // Refresh any artists we added if (addedAuthors.Any()) { _commandQueueManager.Push(new BulkRefreshAuthorCommand(addedAuthors.Select(x => x.Id).ToList(), true)); } return(importResults); }
public void Scan(Artist artist, FilterFilesType filter = FilterFilesType.Known) { var rootFolder = _rootFolderService.GetBestRootFolderPath(artist.Path); if (!_diskProvider.FolderExists(rootFolder)) { _logger.Warn("Artist' root folder ({0}) doesn't exist.", rootFolder); _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(artist, ArtistScanSkippedReason.RootFolderDoesNotExist)); return; } if (_diskProvider.GetDirectories(rootFolder).Empty()) { _logger.Warn("Artist' root folder ({0}) is empty.", rootFolder); _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(artist, ArtistScanSkippedReason.RootFolderIsEmpty)); return; } _logger.ProgressInfo("Scanning {0}", artist.Name); if (!_diskProvider.FolderExists(artist.Path)) { if (_configService.CreateEmptyArtistFolders) { _logger.Debug("Creating missing artist folder: {0}", artist.Path); _diskProvider.CreateFolder(artist.Path); SetPermissions(artist.Path); } else { _logger.Debug("Artist folder doesn't exist: {0}", artist.Path); } CleanMediaFiles(artist, new List <string>()); CompletedScanning(artist); return; } var musicFilesStopwatch = Stopwatch.StartNew(); var mediaFileList = FilterFiles(artist.Path, GetAudioFiles(artist.Path)).ToList(); musicFilesStopwatch.Stop(); _logger.Trace("Finished getting track files for: {0} [{1}]", artist, musicFilesStopwatch.Elapsed); CleanMediaFiles(artist, mediaFileList.Select(x => x.FullName).ToList()); var decisionsStopwatch = Stopwatch.StartNew(); var decisions = _importDecisionMaker.GetImportDecisions(mediaFileList, artist, filter, true); decisionsStopwatch.Stop(); _logger.Debug("Import decisions complete for: {0} [{1}]", artist, decisionsStopwatch.Elapsed); var importStopwatch = Stopwatch.StartNew(); _importApprovedTracks.Import(decisions, false); // decisions may have been filtered to just new files. Anything new and approved will have been inserted. // Now we need to make sure anything new but not approved gets inserted // Note that knownFiles will include anything imported just now var knownFiles = _mediaFileService.GetFilesWithBasePath(artist.Path); var newFiles = decisions .ExceptBy(x => x.Item.Path, knownFiles, x => x.Path, PathEqualityComparer.Instance) .Select(decision => new TrackFile { Path = decision.Item.Path, Size = decision.Item.Size, Modified = decision.Item.Modified, DateAdded = DateTime.UtcNow, Quality = decision.Item.Quality, MediaInfo = decision.Item.FileTrackInfo.MediaInfo }) .ToList(); _mediaFileService.AddMany(newFiles); _logger.Debug($"Inserted {newFiles.Count} new unmatched trackfiles"); // finally update info on size/modified for existing files var updatedFiles = knownFiles .Join(decisions, x => x.Path, x => x.Item.Path, (file, decision) => new { File = file, Item = decision.Item }, PathEqualityComparer.Instance) .Where(x => x.File.Size != x.Item.Size || Math.Abs((x.File.Modified - x.Item.Modified).TotalSeconds) > 1) .Select(x => { x.File.Size = x.Item.Size; x.File.Modified = x.Item.Modified; x.File.MediaInfo = x.Item.FileTrackInfo.MediaInfo; x.File.Quality = x.Item.Quality; return(x.File); }) .ToList(); _mediaFileService.Update(updatedFiles); _logger.Debug($"Updated info for {updatedFiles.Count} known files"); RemoveEmptyArtistFolder(artist.Path); CompletedScanning(artist); importStopwatch.Stop(); _logger.Debug("Track import complete for: {0} [{1}]", artist, importStopwatch.Elapsed); }
public void Scan(List <string> folders = null, FilterFilesType filter = FilterFilesType.Known, bool addNewAuthors = false, List <int> authorIds = null) { if (folders == null) { folders = _rootFolderService.All().Select(x => x.Path).ToList(); } if (authorIds == null) { authorIds = new List <int>(); } var mediaFileList = new List <IFileInfo>(); var musicFilesStopwatch = Stopwatch.StartNew(); foreach (var folder in folders) { // We could be scanning a root folder or a subset of a root folder. If it's a subset, // check if the root folder exists before cleaning. var rootFolder = _rootFolderService.GetBestRootFolder(folder); if (rootFolder == null) { _logger.Error("Not scanning {0}, it's not a subdirectory of a defined root folder", folder); return; } var folderExists = _diskProvider.FolderExists(folder); if (!folderExists) { if (!_diskProvider.FolderExists(rootFolder.Path)) { _logger.Warn("Authors' root folder ({0}) doesn't exist.", rootFolder); var skippedAuthors = _authorService.GetAuthors(authorIds); skippedAuthors.ForEach(x => _eventAggregator.PublishEvent(new AuthorScanSkippedEvent(x, AuthorScanSkippedReason.RootFolderDoesNotExist))); return; } if (_diskProvider.FolderEmpty(rootFolder.Path)) { _logger.Warn("Authors' root folder ({0}) is empty.", rootFolder); var skippedAuthors = _authorService.GetAuthors(authorIds); skippedAuthors.ForEach(x => _eventAggregator.PublishEvent(new AuthorScanSkippedEvent(x, AuthorScanSkippedReason.RootFolderIsEmpty))); return; } } if (!folderExists) { _logger.Debug("Specified scan folder ({0}) doesn't exist.", folder); CleanMediaFiles(folder, new List <string>()); continue; } _logger.ProgressInfo("Scanning {0}", folder); var files = FilterFiles(folder, GetBookFiles(folder)); if (!files.Any()) { _logger.Warn("Scan folder {0} is empty.", folder); continue; } CleanMediaFiles(folder, files.Select(x => x.FullName).ToList()); mediaFileList.AddRange(files); } musicFilesStopwatch.Stop(); _logger.Trace("Finished getting track files for:\n{0} [{1}]", folders.ConcatToString("\n"), musicFilesStopwatch.Elapsed); var decisionsStopwatch = Stopwatch.StartNew(); var config = new ImportDecisionMakerConfig { Filter = filter, IncludeExisting = true, AddNewAuthors = addNewAuthors }; var decisions = _importDecisionMaker.GetImportDecisions(mediaFileList, null, null, config); decisionsStopwatch.Stop(); _logger.Debug("Import decisions complete [{0}]", decisionsStopwatch.Elapsed); var importStopwatch = Stopwatch.StartNew(); _importApprovedTracks.Import(decisions, false); // decisions may have been filtered to just new files. Anything new and approved will have been inserted. // Now we need to make sure anything new but not approved gets inserted // Note that knownFiles will include anything imported just now var knownFiles = new List <BookFile>(); folders.ForEach(x => knownFiles.AddRange(_mediaFileService.GetFilesWithBasePath(x))); var newFiles = decisions .ExceptBy(x => x.Item.Path, knownFiles, x => x.Path, PathEqualityComparer.Instance) .Select(decision => new BookFile { Path = decision.Item.Path, CalibreId = decision.Item.Path.ParseCalibreId(), Size = decision.Item.Size, Modified = decision.Item.Modified, DateAdded = DateTime.UtcNow, Quality = decision.Item.Quality, MediaInfo = decision.Item.FileTrackInfo.MediaInfo, Edition = decision.Item.Edition }) .ToList(); _mediaFileService.AddMany(newFiles); _logger.Debug($"Inserted {newFiles.Count} new unmatched trackfiles"); // finally update info on size/modified for existing files var updatedFiles = knownFiles .Join(decisions, x => x.Path, x => x.Item.Path, (file, decision) => new { File = file, Item = decision.Item }, PathEqualityComparer.Instance) .Where(x => x.File.Size != x.Item.Size || Math.Abs((x.File.Modified - x.Item.Modified).TotalSeconds) > 1) .Select(x => { x.File.Size = x.Item.Size; x.File.Modified = x.Item.Modified; x.File.MediaInfo = x.Item.FileTrackInfo.MediaInfo; x.File.Quality = x.Item.Quality; return(x.File); }) .ToList(); _mediaFileService.Update(updatedFiles); _logger.Debug($"Updated info for {updatedFiles.Count} known files"); var authors = _authorService.GetAuthors(authorIds); foreach (var author in authors) { CompletedScanning(author); } importStopwatch.Stop(); _logger.Debug("Book import complete for:\n{0} [{1}]", folders.ConcatToString("\n"), importStopwatch.Elapsed); }
public List <ImportResult> Import(List <ImportDecision <LocalTrack> > decisions, bool replaceExisting, DownloadClientItem downloadClientItem = null, ImportMode importMode = ImportMode.Auto) { var qualifiedImports = decisions.Where(c => c.Approved) .GroupBy(c => c.Item.Artist.Id, (i, s) => s .OrderByDescending(c => c.Item.Quality, new QualityModelComparer(s.First().Item.Artist.QualityProfile)) .ThenByDescending(c => c.Item.Size)) .SelectMany(c => c) .ToList(); _logger.Debug($"Importing {qualifiedImports.Count} files. replaceExisting: {replaceExisting}"); var importResults = new List <ImportResult>(); var allImportedTrackFiles = new List <TrackFile>(); var allOldTrackFiles = new List <TrackFile>(); var albumDecisions = decisions.Where(e => e.Item.Album != null && e.Approved) .GroupBy(e => e.Item.Album.Id).ToList(); foreach (var albumDecision in albumDecisions) { var album = albumDecision.First().Item.Album; var newRelease = albumDecision.First().Item.Release; if (replaceExisting) { var artist = albumDecision.First().Item.Artist; var rootFolder = _diskProvider.GetParentFolder(artist.Path); var previousFiles = _mediaFileService.GetFilesByAlbum(album.Id); _logger.Debug($"Deleting {previousFiles.Count} existing files for {album}"); foreach (var previousFile in previousFiles) { var subfolder = rootFolder.GetRelativePath(_diskProvider.GetParentFolder(previousFile.Path)); if (_diskProvider.FileExists(previousFile.Path)) { _logger.Debug("Removing existing track file: {0}", previousFile); _recycleBinProvider.DeleteFile(previousFile.Path, subfolder); } _mediaFileService.Delete(previousFile, DeleteMediaFileReason.Upgrade); } } // set the correct release to be monitored before importing the new files _logger.Debug("Updating release to {0} [{1} tracks]", newRelease, newRelease.TrackCount); album.AlbumReleases = _releaseService.SetMonitored(newRelease); // Publish album edited event. // Deliberatly don't put in the old album since we don't want to trigger an ArtistScan. _eventAggregator.PublishEvent(new AlbumEditedEvent(album, album)); } var filesToAdd = new List <TrackFile>(qualifiedImports.Count); var albumReleasesDict = new Dictionary <int, List <AlbumRelease> >(albumDecisions.Count); var trackImportedEvents = new List <TrackImportedEvent>(qualifiedImports.Count); foreach (var importDecision in qualifiedImports.OrderBy(e => e.Item.Tracks.Select(track => track.AbsoluteTrackNumber).MinOrDefault()) .ThenByDescending(e => e.Item.Size)) { var localTrack = importDecision.Item; var oldFiles = new List <TrackFile>(); try { //check if already imported if (importResults.SelectMany(r => r.ImportDecision.Item.Tracks) .Select(e => e.Id) .Intersect(localTrack.Tracks.Select(e => e.Id)) .Any()) { importResults.Add(new ImportResult(importDecision, "Track has already been imported")); continue; } // cache album releases and set artist to speed up firing the TrackImported events // (otherwise they'll be retrieved from the DB for each track) if (!albumReleasesDict.ContainsKey(localTrack.Album.Id)) { albumReleasesDict.Add(localTrack.Album.Id, localTrack.Album.AlbumReleases.Value); } if (!localTrack.Album.AlbumReleases.IsLoaded) { localTrack.Album.AlbumReleases = albumReleasesDict[localTrack.Album.Id]; } localTrack.Album.Artist = localTrack.Artist; foreach (var track in localTrack.Tracks) { track.Artist = localTrack.Artist; track.AlbumRelease = localTrack.Release; track.Album = localTrack.Album; } var trackFile = new TrackFile { Path = localTrack.Path.CleanFilePath(), Size = localTrack.Size, Modified = localTrack.Modified, DateAdded = DateTime.UtcNow, ReleaseGroup = localTrack.ReleaseGroup, Quality = localTrack.Quality, MediaInfo = localTrack.FileTrackInfo.MediaInfo, AlbumId = localTrack.Album.Id, Artist = localTrack.Artist, Album = localTrack.Album, Tracks = localTrack.Tracks }; bool copyOnly; switch (importMode) { default: case ImportMode.Auto: copyOnly = downloadClientItem != null && !downloadClientItem.CanMoveFiles; break; case ImportMode.Move: copyOnly = false; break; case ImportMode.Copy: copyOnly = true; break; } if (!localTrack.ExistingFile) { trackFile.SceneName = GetSceneReleaseName(downloadClientItem, localTrack); var moveResult = _trackFileUpgrader.UpgradeTrackFile(trackFile, localTrack, copyOnly); oldFiles = moveResult.OldFiles; } else { // Delete existing files from the DB mapped to this path var previousFile = _mediaFileService.GetFileWithPath(trackFile.Path); if (previousFile != null) { _mediaFileService.Delete(previousFile, DeleteMediaFileReason.ManualOverride); } _audioTagService.WriteTags(trackFile, false); } filesToAdd.Add(trackFile); importResults.Add(new ImportResult(importDecision)); if (!localTrack.ExistingFile) { _extraService.ImportTrack(localTrack, trackFile, copyOnly); } allImportedTrackFiles.Add(trackFile); allOldTrackFiles.AddRange(oldFiles); // create all the import events here, but we can't publish until the trackfiles have been // inserted and ids created trackImportedEvents.Add(new TrackImportedEvent(localTrack, trackFile, oldFiles, !localTrack.ExistingFile, downloadClientItem)); } catch (RootFolderNotFoundException e) { _logger.Warn(e, "Couldn't import track " + localTrack); _eventAggregator.PublishEvent(new TrackImportFailedEvent(e, localTrack, !localTrack.ExistingFile, downloadClientItem)); importResults.Add(new ImportResult(importDecision, "Failed to import track, Root folder missing.")); } catch (DestinationAlreadyExistsException e) { _logger.Warn(e, "Couldn't import track " + localTrack); importResults.Add(new ImportResult(importDecision, "Failed to import track, Destination already exists.")); } catch (UnauthorizedAccessException e) { _logger.Warn(e, "Couldn't import track " + localTrack); _eventAggregator.PublishEvent(new TrackImportFailedEvent(e, localTrack, !localTrack.ExistingFile, downloadClientItem)); importResults.Add(new ImportResult(importDecision, "Failed to import track, Permissions error")); } catch (Exception e) { _logger.Warn(e, "Couldn't import track " + localTrack); importResults.Add(new ImportResult(importDecision, "Failed to import track")); } } var watch = new System.Diagnostics.Stopwatch(); watch.Start(); _mediaFileService.AddMany(filesToAdd); _logger.Debug($"Inserted new trackfiles in {watch.ElapsedMilliseconds}ms"); filesToAdd.ForEach(f => f.Tracks.Value.ForEach(t => t.TrackFileId = f.Id)); _trackService.SetFileIds(filesToAdd.SelectMany(x => x.Tracks.Value).ToList()); _logger.Debug($"TrackFileIds updated, total {watch.ElapsedMilliseconds}ms"); // now that trackfiles have been inserted and ids generated, publish the import events foreach (var trackImportedEvent in trackImportedEvents) { _eventAggregator.PublishEvent(trackImportedEvent); } var albumImports = importResults.Where(e => e.ImportDecision.Item.Album != null) .GroupBy(e => e.ImportDecision.Item.Album.Id).ToList(); foreach (var albumImport in albumImports) { var release = albumImport.First().ImportDecision.Item.Release; var album = albumImport.First().ImportDecision.Item.Album; var artist = albumImport.First().ImportDecision.Item.Artist; if (albumImport.Where(e => e.Errors.Count == 0).ToList().Count > 0 && artist != null && album != null) { _eventAggregator.PublishEvent(new AlbumImportedEvent( artist, album, release, allImportedTrackFiles.Where(s => s.AlbumId == album.Id).ToList(), allOldTrackFiles.Where(s => s.AlbumId == album.Id).ToList(), replaceExisting, downloadClientItem)); } } //Adding all the rejected decisions importResults.AddRange(decisions.Where(c => !c.Approved) .Select(d => new ImportResult(d, d.Rejections.Select(r => r.Reason).ToArray()))); return(importResults); }