public DeletionApiController(VolyContext context, IDapperConnection dapper, VSettings settings, ILogger <ConversionApiController> logger)
{
db = context;
this.dapper = dapper;
this.settings = settings;
log = logger;
}
/// <summary>
///
/// </summary>
public PostConversionPluginArgs(ILibrary library, VolyContext context, IConversionItem item, MediaItem mediaItem, DashEncodeResult conversionResult, ConversionType type, ILogger log)
{
this.Library = library;
this.Context = context;
this.ConversionItem = item;
this.MediaItem = mediaItem;
this.ConversionResult = conversionResult;
this.Type = type;
this.Log = log;
}
public void FullScan()
{
log.LogInformation($"Full scan requested.");
using (var outerContext = new VolyContext(dbOptions))
{
foreach (var library in settings.Libraries)
{
scanner.ScheduleLibraryScan(library, library.StorageBackend.RetrieveBackend(), outerContext);
}
}
}
public IActionResult LibraryScan(string libraryName, [FromBody] List <string> scanFilter)
{
log.LogInformation($"Scan of library {libraryName} requested.");
var library = settings.Libraries.Where(x => x.Name == libraryName).FirstOrDefault();
if (library != null)
{
using (var outerContext = new VolyContext(dbOptions))
{
scanner.ScheduleLibraryScan(library, library.StorageBackend.RetrieveBackend(), outerContext, scanFilter);
}
return(Ok());
}
else
{
return(BadRequest());
}
}
public void TestMediaItem()
{
using (var dbConnection = GetDatabase())
{
var dbBuilder = new DbContextOptionsBuilder <VolyContext>().UseSqlite(dbConnection);
using (var context = new VolyContext(dbBuilder.Options))
{
VolySeed.Initialize(context, null);
var newMedia = new MediaItem()
{
SourcePath = "",
SourceModified = DateTime.Now,
SourceHash = "",
Duration = TimeSpan.FromSeconds(1),
IndexName = "",
IndexHash = "",
LibraryName = "Library",
Name = "",
SeriesName = ""
};
context.Media.Add(newMedia);
context.SaveChanges();
Assert.AreEqual(1, newMedia.MediaId);
context.TransactionLog.Add(new TransactionLog()
{
TableName = VolyExports.TransactionTableType.MediaItem,
Type = VolyExports.TransactionType.Insert,
Key = newMedia.MediaId
});
context.SaveChanges();
Assert.AreEqual(1, context.TransactionLog.FirstOrDefaultAsync().Result.Key);
}
}
}
public void TestConversion()
{
var globalTemp = Path.Join(Environment.CurrentDirectory, "testing\\globaltemp");
Directory.CreateDirectory(globalTemp);
var inputDirectory = Path.Join(Environment.CurrentDirectory, "testing\\input");
Directory.CreateDirectory(inputDirectory);
if (!File.Exists("testing\\input\\test5.mkv"))
{
File.Copy(Path.Join(Environment.CurrentDirectory, "..\\..\\..\\test5.mkv"), "testing\\input\\test5.mkv");
}
if (!File.Exists("testing\\input\\test52.mkv"))
{
File.Copy(Path.Join(Environment.CurrentDirectory, "..\\..\\..\\test52.mkv"), "testing\\input\\test52.mkv");
}
if (!File.Exists("testing\\input\\testfile.ogg"))
{
File.Copy(Path.Join(Environment.CurrentDirectory, "..\\..\\..\\testfile.ogg"), "testing\\input\\testfile.ogg");
}
if (!File.Exists("testing\\input\\testfile2.ogg"))
{
File.Copy(Path.Join(Environment.CurrentDirectory, "..\\..\\..\\testfile2.ogg"), "testing\\input\\testfile2.ogg");
}
var libraryTemp = Path.Join(Environment.CurrentDirectory, "testing\\librarytemp");
Directory.CreateDirectory(libraryTemp);
var outputDirectory = Path.Join(Environment.CurrentDirectory, "testing\\output");
Directory.CreateDirectory(outputDirectory);
string litePath = Path.Combine(Environment.CurrentDirectory, "temp.sqlite");
if (File.Exists(litePath))
{
File.Delete(litePath);
}
var connection = new SqliteConnection("Data Source=:memory:");
connection.Open();
try
{
var dbBuilder = new DbContextOptionsBuilder <VolyContext>()
.UseSqlite(connection);
MediaDatabase db = new MediaDatabase
{
Database = dbBuilder.Options
};
var logFactory = new LoggerFactory();
DQP.IDistinctQueueProcessor <IConversionItem> converter = new MediaConversionQueue(
"ffmpeg",
"ffprobe",
"mp4box",
globalTemp,
1,
new CompleteItems <IExportableConversionItem>(),
new Logger <MediaConversionQueue>(logFactory));
var scanQueue = new LibraryScanningQueue(db, converter, new List <IConversionPlugin>(), new Logger <LibraryScanningQueue>(logFactory));
var quality1 = new Quality(640, 480, 300, DEnc.H264Preset.ultrafast);
var quality2 = new Quality(640, 480, 400, DEnc.H264Preset.ultrafast);
var testLibrary = new Library()
{
Name = "Test",
OriginPath = inputDirectory,
Qualities = new List <Quality>()
{
quality1, quality2
},
ValidExtensions = new HashSet <string>()
{
".mp4", ".mkv", ".ogg"
},
TempPath = libraryTemp
};
IStorage storage = new MStorage.FilesystemStorage.FilesystemStorage(outputDirectory);
using var context = new VolyContext(dbBuilder.Options);
VolySeed.Initialize(context, logFactory.CreateLogger("VolySeed"));
scanQueue.ScheduleLibraryScan(testLibrary, storage, context);
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
while (converter.ItemsQueued.Count == 0)
{
if (sw.ElapsedMilliseconds > 10000)
{
Assert.Fail("Disk scan took too long.");
}
System.Threading.Thread.Sleep(10);
}
var itemsProcessed = new Dictionary <string, IConversionItem>();
int lastProgressesSeen = 0;
int progressMovedEvents = 0;
while (converter.ItemsQueued.Count > 0)
{
foreach (var item in converter.ItemsProcessing)
{
Assert.IsTrue(item.Value.Quality.Where(x => x.Bitrate == quality1.Bitrate).SingleOrDefault() != null, "Item does not have the correct quality1 configuration.");
Assert.IsTrue(item.Value.Quality.Where(x => x.Bitrate == quality2.Bitrate).SingleOrDefault() != null, "Item does not have the correct quality2 configuration.");
if (!itemsProcessed.ContainsKey(item.Key))
{
itemsProcessed.Add(item.Key, item.Value.DeepClone());
Assert.AreEqual(item.Value.SourcePath, itemsProcessed[item.Key].SourcePath);
}
else
{
int progressesSeen = 0;
var updateProgresses = item.Value.Progress.GetEnumerator();
var currentProgresses = itemsProcessed[item.Key].Progress.GetEnumerator();
while (updateProgresses.MoveNext())
{
progressesSeen++;
if (currentProgresses.MoveNext())
{
Assert.AreEqual(currentProgresses.Current.Description, updateProgresses.Current.Description);
Assert.IsTrue(updateProgresses.Current.Progress >= currentProgresses.Current.Progress, "Progress not expected to go backwards.");
if (updateProgresses.Current.Progress > currentProgresses.Current.Progress)
{
progressMovedEvents++;
}
}
}
lastProgressesSeen = progressesSeen;
item.Value.DeepCloneTo(itemsProcessed[item.Key]);
}
}
System.Threading.Thread.Sleep(10);
}
Assert.IsTrue(lastProgressesSeen > 1);
Assert.IsTrue(progressMovedEvents > 1);
}
finally
{
connection.Close();
}
}
public MediaQueryController(VolyContext context, IDapperConnection dapper, ILogger <ScannerController> logger)
{
db = context;
this.dapper = dapper;
log = logger;
}
public void ScheduleLibraryScan(Library library, IStorage storageBackend, VolyContext context, IEnumerable <string> scanTheseFilesOnly)
{
AddItem(new LibraryScanner(library, storageBackend, converter, dbOptions.Database, conversionPlugins, log, scanTheseFilesOnly));
}
public void ScheduleLibraryScan(Library library, IStorage storageBackend, VolyContext context)
{
AddItem(new LibraryScanner(library, storageBackend, converter, dbOptions.Database, conversionPlugins, log));
}
public override bool Scan()
{
if (!library.Enable)
{
log.LogInformation($"Library {library.Name} skipped (disabled).");
return(false);
}
else if (CancellationToken.IsCancellationRequested)
{
log.LogInformation($"Library {library.Name} skipped (cancelled).");
return(false);
}
else if (!Directory.Exists(library.OriginPath))
{
log.LogWarning($"Scanning library {library.Name} failed: Directory does not exist.");
return(false);
}
using var context = new VolyContext(dbOptions);
IEnumerable <VolyDatabase.MediaItem> contextMediaItems;
contextMediaItems = context.Media
.Where(x => x.LibraryName == library.Name)
.Select(x => new VolyDatabase.MediaItem()
{
MediaId = x.MediaId, SourcePath = x.SourcePath, SourceModified = x.SourceModified, SourceHash = x.SourceHash, IndexHash = x.IndexHash
});
if (scanTheseFilesOnly != null)
{
// Reduce the data set to only items which are in our scan selection.
var extensionlessSTFO = scanTheseFilesOnly.Select(x => Path.GetFileNameWithoutExtension(x)).ToHashSet();
contextMediaItems = contextMediaItems.Where(x => extensionlessSTFO.Contains(Path.GetFileNameWithoutExtension(x.SourcePath)));
}
var currentLibrary = new Dictionary <string, VolyDatabase.MediaItem>();
var dbMediaItemsNotFound = new HashSet <int>();
foreach (var item in contextMediaItems)
{
currentLibrary[Path.GetFileNameWithoutExtension(item.SourcePath)] = item;
dbMediaItemsNotFound.Add(item.MediaId);
}
var quality = new HashSet <IQuality>(library.Qualities);
IEnumerable <ScanFile> foundFiles;
if (scanTheseFilesOnly == null)
{
log.LogInformation($"Scanning library {library.Name}...");
foundFiles = GetLibraryFiles(library.OriginPath);
}
else if (scanTheseFilesOnly.Count() == 0)
{
log.LogWarning("Library scan was started with a specified scan set, but the scan set was empty.");
return(false);
}
else
{
log.LogInformation($"Scanning library {library.Name} for {scanTheseFilesOnly.Count()} files...");
foundFiles = GetLibraryFiles(library.OriginPath, scanTheseFilesOnly);
}
foreach (var file in foundFiles)
{
if (CancellationToken.IsCancellationRequested)
{
log.LogInformation($"Halted scan of library {library.Name} (cancelled).");
return(false);
}
var extension = Path.GetExtension(file.Path);
if (!library.ValidExtensions.Contains(extension))
{
continue;
}
bool existsInDb = currentLibrary.TryGetValue(Path.GetFileNameWithoutExtension(file.Path), out var existingEntry);
if (existsInDb)
{
dbMediaItemsNotFound.Remove(existingEntry.MediaId);
} // Remove to track missing entries for later.
string seriesName = new DirectoryInfo(Directory.GetParent(file.Path).FullName).Name;
if (file.ZeroLength)
{
continue;
}
else if (existsInDb && string.IsNullOrWhiteSpace(existingEntry.IndexHash)) // Indicates previous failed conversion.
{
log.LogInformation($"Scheduling conversion of previously failed item at path {file.Path}");
using (var deletionContext = new VolyContext(dbOptions))
{
deletionContext.Media.Remove(existingEntry);
deletionContext.SaveChanges();
}
ScheduleConversion(library, quality, file.Path, file.LastModified, seriesName, file);
}
else if (existsInDb)
{
TryReconvert(quality, file, existingEntry, seriesName);
}
else
{
var oldVersion = context.Media.Where(x => x.SourceHash == file.SourceHash).SingleOrDefault();
if (oldVersion != null)
{
log.LogInformation($"Discovered renamed file. {oldVersion.SourcePath} => {file.Path}");
dbMediaItemsNotFound.Remove(oldVersion.MediaId);
oldVersion.SourcePath = file.Path;
context.Media.Update(oldVersion);
if (!TryReconvert(quality, file, oldVersion, seriesName))
{
// Not being reconverted, just update metadata.
RunPrePlugins(library, null, oldVersion, ConversionType.MetadataOnly);
context.Media.Update(oldVersion);
RunPostPlugins(library, context, null, oldVersion, null, ConversionType.MetadataOnly);
}
}
else
{
// Media doesn't exist in db, make it.
log.LogInformation($"Scheduling conversion of {file.Path}");
ScheduleConversion(library, quality, file.Path, file.LastModified, seriesName, file);
}
}
}
if (library.DeleteWithSource)
{
// Queue delete for items not found in scan.
var inDb = context.PendingDeletions.Where(x => dbMediaItemsNotFound.Contains(x.MediaId)).Select(y => y.MediaId);
dbMediaItemsNotFound.ExceptWith(inDb);
context.PendingDeletions.AddRange(dbMediaItemsNotFound.Select(x => new PendingDeletion()
{
MediaId = x, Version = -1, Requestor = DeleteRequester.Scan
}));
}
context.SaveChanges();
log.LogInformation($"Scanning library {library.Name} complete.");
return(true);
}
