/// <summary>
/// Gets all files known by this test writer.
/// </summary>
/// <returns>All files known by this test writer.</returns>
public IEnumerable <Tuple <string, string> > GetAllFiles()
{
var files = new List <Tuple <string, string> >();
files.AddRange(ConfigWriter.GetAllFiles());
files.AddRange(ExtraFiles.Select(kvp => Tuple.Create(kvp.Key, kvp.Value)));
return(files);
}
public void Add(List <Identifier> idents, string relPath)
{
var ident = idents.FirstOrDefault();
if (ident == null)
{
ExtraFiles.Add(relPath);
return;
}
if (ident is SkinIdentifier skinId)
{
var skinIds = idents.Cast <SkinIdentifier>().ToList();
if (skinIds.Count > 1)
{
MultiSkinFiles.Add(relPath, skinIds);
}
else
{
Skins.Add(relPath, ident as SkinIdentifier);
}
}
else if (ident is PortraitIdentifier)
{
Portraits.Add(relPath, idents.Cast <PortraitIdentifier>());
}
else if (ident is CrosshairIdentifier)
{
Crosshairs.Add(relPath, idents.Cast <CrosshairIdentifier>());
}
else if (ident is WeaponIdentifier)
{
Weapons.Add(relPath, idents.Cast <WeaponIdentifier>());
}
else if (ident is EffectsIdentifier)
{
Effects.Add(relPath, idents.Cast <EffectsIdentifier>());
}
else if (ident is CanopyIdentifier)
{
Canopies.Add(relPath, idents.Cast <CanopyIdentifier>());
}
else if (ident is EmblemIdentifier)
{
Emblems.Add(relPath, idents.Cast <EmblemIdentifier>());
}
else if (ident is CockpitIdentifier)
{
Cockpits.Add(relPath, idents.Cast <CockpitIdentifier>());
}
}
/// <summary>
/// This function will search for a way to create a BSA in the installed mod list by assembling it from files
/// found in archives. To do this we hash all the files in side the BSA then try to find matches and patches for
/// all of the files.
/// </summary>
/// <returns></returns>
private Func <RawSourceFile, Directive> DeconstructBSAs()
{
var include_directly = ModInis.Where(kv => {
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE))
{
return(true);
}
if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE))
{
return(true);
}
return(false);
}).Select(kv => $"mods\\{kv.Key}\\");
var microstack = new List <Func <RawSourceFile, Directive> >()
{
DirectMatch(),
IncludePatches(),
DropAll()
};
var microstack_with_include = new List <Func <RawSourceFile, Directive> >()
{
DirectMatch(),
IncludePatches(),
IncludeALL()
};
return(source =>
{
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path)))
{
return null;
}
bool default_include = false;
if (source.Path.StartsWith("mods"))
{
foreach (var modpath in include_directly)
{
if (source.Path.StartsWith(modpath))
{
default_include = true;
break;
}
}
}
var source_files = source.File.FileInArchive;
var stack = default_include ? microstack_with_include : microstack;
var id = Guid.NewGuid().ToString();
var matches = source_files.PMap(e => RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last())
}));
foreach (var match in matches)
{
if (match is IgnoredDirectly)
{
Error($"File required for BSA creation doesn't exist: {match.To}");
}
ExtraFiles.Add(match);
}
;
CreateBSA directive;
using (var bsa = new BSAReader(source.AbsolutePath))
{
directive = new CreateBSA()
{
To = source.Path,
TempID = id,
Type = (uint)bsa.HeaderType,
FileFlags = (uint)bsa.FileFlags,
ArchiveFlags = (uint)bsa.ArchiveFlags,
};
};
return directive;
});
}
public async Task <Either <BaseError, Unit> > ScanFolder(LibraryPath libraryPath, string ffprobePath)
{
if (!_localFileSystem.IsLibraryPathAccessible(libraryPath))
{
return(new MediaSourceInaccessible());
}
var folderQueue = new Queue <string>();
foreach (string folder in _localFileSystem.ListSubdirectories(libraryPath.Path).OrderBy(identity))
{
folderQueue.Enqueue(folder);
}
while (folderQueue.Count > 0)
{
string movieFolder = folderQueue.Dequeue();
var allFiles = _localFileSystem.ListFiles(movieFolder)
.Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f)))
.Filter(
f => !ExtraFiles.Any(
e => Path.GetFileNameWithoutExtension(f).EndsWith(e, StringComparison.OrdinalIgnoreCase)))
.ToList();
if (allFiles.Count == 0)
{
foreach (string subdirectory in _localFileSystem.ListSubdirectories(movieFolder).OrderBy(identity))
{
folderQueue.Enqueue(subdirectory);
}
continue;
}
foreach (string file in allFiles.OrderBy(identity))
{
// TODO: figure out how to rebuild playlists
Either <BaseError, MediaItemScanResult <Movie> > maybeMovie = await _movieRepository
.GetOrAdd(libraryPath, file)
.BindT(movie => UpdateStatistics(movie, ffprobePath))
.BindT(UpdateMetadata)
.BindT(movie => UpdateArtwork(movie, ArtworkKind.Poster))
.BindT(movie => UpdateArtwork(movie, ArtworkKind.FanArt));
await maybeMovie.Match(
async result =>
{
if (result.IsAdded)
{
await _searchIndex.AddItems(new List <MediaItem> {
result.Item
});
}
else if (result.IsUpdated)
{
await _searchIndex.UpdateItems(new List <MediaItem> {
result.Item
});
}
},
error =>
{
_logger.LogWarning("Error processing movie at {Path}: {Error}", file, error.Value);
return(Task.CompletedTask);
});
}
}
foreach (string path in await _movieRepository.FindMoviePaths(libraryPath))
{
if (!_localFileSystem.FileExists(path))
{
_logger.LogInformation("Removing missing movie at {Path}", path);
List <int> ids = await _movieRepository.DeleteByPath(libraryPath, path);
await _searchIndex.RemoveItems(ids);
}
}
return(Unit.Default);
}
public async Task <Either <BaseError, Unit> > ScanFolder(
LibraryPath libraryPath,
string ffmpegPath,
string ffprobePath,
decimal progressMin,
decimal progressMax,
CancellationToken cancellationToken)
{
try
{
decimal progressSpread = progressMax - progressMin;
var foldersCompleted = 0;
var folderQueue = new Queue <string>();
foreach (string folder in _localFileSystem.ListSubdirectories(libraryPath.Path)
.Filter(ShouldIncludeFolder)
.OrderBy(identity))
{
folderQueue.Enqueue(folder);
}
while (folderQueue.Count > 0)
{
if (cancellationToken.IsCancellationRequested)
{
return(new ScanCanceled());
}
decimal percentCompletion = (decimal)foldersCompleted / (foldersCompleted + folderQueue.Count);
await _mediator.Publish(
new LibraryScanProgress(libraryPath.LibraryId, progressMin + percentCompletion *progressSpread),
cancellationToken);
string movieFolder = folderQueue.Dequeue();
foldersCompleted++;
var filesForEtag = _localFileSystem.ListFiles(movieFolder).ToList();
var allFiles = filesForEtag
.Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f)))
.Filter(f => !Path.GetFileName(f).StartsWith("._"))
.Filter(
f => !ExtraFiles.Any(
e => Path.GetFileNameWithoutExtension(f).EndsWith(e, StringComparison.OrdinalIgnoreCase)))
.ToList();
if (allFiles.Count == 0)
{
foreach (string subdirectory in _localFileSystem.ListSubdirectories(movieFolder)
.Filter(ShouldIncludeFolder)
.OrderBy(identity))
{
folderQueue.Enqueue(subdirectory);
}
continue;
}
string etag = FolderEtag.Calculate(movieFolder, _localFileSystem);
Option <LibraryFolder> knownFolder = libraryPath.LibraryFolders
.Filter(f => f.Path == movieFolder)
.HeadOrNone();
// skip folder if etag matches
if (await knownFolder.Map(f => f.Etag ?? string.Empty).IfNoneAsync(string.Empty) == etag)
{
continue;
}
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
movieFolder);
foreach (string file in allFiles.OrderBy(identity))
{
// TODO: figure out how to rebuild playlists
Either <BaseError, MediaItemScanResult <Movie> > maybeMovie = await _movieRepository
.GetOrAdd(libraryPath, file)
.BindT(movie => UpdateStatistics(movie, ffmpegPath, ffprobePath))
.BindT(UpdateMetadata)
.BindT(movie => UpdateArtwork(movie, ArtworkKind.Poster, cancellationToken))
.BindT(movie => UpdateArtwork(movie, ArtworkKind.FanArt, cancellationToken))
.BindT(UpdateSubtitles)
.BindT(FlagNormal);
foreach (BaseError error in maybeMovie.LeftToSeq())
{
_logger.LogWarning("Error processing movie at {Path}: {Error}", file, error.Value);
}
foreach (MediaItemScanResult <Movie> result in maybeMovie.RightToSeq())
{
if (result.IsAdded)
{
await _searchIndex.AddItems(_searchRepository, new List <MediaItem> {
result.Item
});
}
else if (result.IsUpdated)
{
await _searchIndex.UpdateItems(_searchRepository, new List <MediaItem> {
result.Item
});
}
await _libraryRepository.SetEtag(libraryPath, knownFolder, movieFolder, etag);
}
}
}
foreach (string path in await _movieRepository.FindMoviePaths(libraryPath))
{
if (!_localFileSystem.FileExists(path))
{
_logger.LogInformation("Flagging missing movie at {Path}", path);
List <int> ids = await FlagFileNotFound(libraryPath, path);
await _searchIndex.RebuildItems(_searchRepository, ids);
}
else if (Path.GetFileName(path).StartsWith("._"))
{
_logger.LogInformation("Removing dot underscore file at {Path}", path);
List <int> ids = await _movieRepository.DeleteByPath(libraryPath, path);
await _searchIndex.RemoveItems(ids);
}
}
await _libraryRepository.CleanEtagsForLibraryPath(libraryPath);
return(Unit.Default);
}
catch (Exception ex) when(ex is TaskCanceledException or OperationCanceledException)
{
return(new ScanCanceled());
}
finally
{
_searchIndex.Commit();
}
}
