public async Task Sync()
{
if (!await m_GoogleBearerTokenRetriever.Init())
{
return;
}
m_Logger.LogInformation("Loading Google Albums");
await m_GoogleSource.Load();
m_Logger.LogInformation($"Total Albums: {m_GoogleSource.Albums.Count}");
m_Logger.LogInformation("Loading local collection");
m_LocalSource.Load();
m_Logger.LogInformation($"Total Albums: {m_LocalSource.PhotoAlbums.Count}");
m_Logger.LogInformation($"Total Files: {m_LocalSource.TotalFiles}");
m_Logger.LogInformation($"Total Size: {m_LocalSource.TotalBytes.AsHumanReadableBytes("MB")}");
m_Logger.LogInformation("Comparing");
var collectionDiff = new CollectionDiff(m_LocalSource, m_GoogleSource);
m_Logger.LogInformation(collectionDiff.ToString());
m_Logger.LogInformation("Syncing");
await m_CollectionSync.SyncCollection(collectionDiff);
}
private void AssertNames(CollectionDiff <LocalizedStringWithId, LocalizedStringWithId> result,
IEnumerable <LocalizedStringWithIdContract> added = null,
IEnumerable <LocalizedStringWithIdContract> removed = null, IEnumerable <LocalizedStringWithIdContract> unchanged = null)
{
AssertCollection(result.Added, added, "added");
AssertCollection(result.Removed, removed, "removed");
AssertCollection(result.Unchanged, unchanged, "unchanged");
}
public void SyncLocalFilePVs(CollectionDiff <PVForSong, PVForSong> diff, int songId)
{
var addedLocalMedia = diff.Added.Where(m => m.Service == PVService.LocalFile);
foreach (var pv in addedLocalMedia)
{
var oldFull = Path.Combine(Path.GetTempPath(), pv.PVId);
if (Path.GetDirectoryName(oldFull) != Path.GetDirectoryName(Path.GetTempPath()))
{
throw new InvalidOperationException("File folder doesn't match with temporary folder");
}
if (!Extensions.Contains(Path.GetExtension(oldFull)))
{
throw new InvalidOperationException("Invalid extension");
}
var newId = $"{pv.Author}-S{songId}-{pv.PVId}";
var newFull = GetFilesystemPath(newId);
pv.PVId = newId;
try
{
File.Move(oldFull, newFull);
// Remove copied permissions, reset to inherited http://stackoverflow.com/a/2930969
var newFullFileInfo = new FileInfo(newFull);
var fs = newFullFileInfo.GetAccessControl();
fs.SetAccessRuleProtection(false, false);
newFullFileInfo.SetAccessControl(fs);
CreateThumbnail(newFull, newId, pv);
}
catch (IOException x)
{
s_log.Error(x, "Unable to move local media file: " + oldFull);
throw;
}
}
foreach (var pv in diff.Removed.Where(m => m.Service == PVService.LocalFile))
{
var fullPath = GetFilesystemPath(pv.PVId);
if (File.Exists(fullPath))
{
try
{
File.Delete(fullPath);
}
catch (IOException x)
{
s_log.Error(x, "Unable to delete local media file: " + fullPath);
}
}
}
}
public void CalculateNewItemIndex_WithNoBeforeOrAfterIdFound_ReturnsIndexAtEndOfList()
{
var change = new CollectionDiff()
{
Id = "C",
NowBeforeId = "D"
};
var items = new List<KeyValuePair<string, string>>
{
new KeyValuePair<string, string>("A", ""),
new KeyValuePair<string, string>("B", "")
};
int newIndex = OrderedCollectionDiffMerge.CalculateNewItemIndex(change, items);
Assert.AreEqual(2, newIndex);
}
public void CalculateNewItemIndex_WithBeforeIdSet_ReturnsIndexOfBeforeId()
{
var change = new CollectionDiff()
{
Id = "B",
NowBeforeId = "A"
};
var items = new List<KeyValuePair<string, string>>
{
new KeyValuePair<string, string>("A", ""),
new KeyValuePair<string, string>("B", "")
};
int newIndex = OrderedCollectionDiffMerge.CalculateNewItemIndex(change, items);
Assert.AreEqual(0, newIndex);
}
public static void Sync <T>(ISession session, CollectionDiff <T, T> diff)
{
ParamIs.NotNull(() => session);
ParamIs.NotNull(() => diff);
foreach (var n in diff.Removed)
{
session.Delete(n);
}
foreach (var n in diff.Added)
{
session.Save(n);
}
foreach (var n in diff.Unchanged)
{
session.Update(n);
}
}
public static void Sync <T>(this IDatabaseContext <T> ctx, CollectionDiff <T, T> diff)
{
ParamIs.NotNull(() => ctx);
ParamIs.NotNull(() => diff);
foreach (var n in diff.Removed)
{
ctx.Delete(n);
}
foreach (var n in diff.Added)
{
ctx.Save(n);
}
foreach (var n in diff.Unchanged)
{
ctx.Update(n);
}
}
public static CollectionDiff <T2, T2> Sync <T, T2>(this IDatabaseContext <T> ctx, CollectionDiff <T2, T2> diff)
{
ParamIs.NotNull(() => ctx);
Sync <T2>(ctx.OfType <T2>(), diff);
return(diff);
}
public void MergeChangeIntoOrderedItems_WithRenameChange_RenamesTheItem()
{
var orderedItems = new List<KeyValuePair<string, string>>
{
new KeyValuePair<string, string>("A", "1"),
new KeyValuePair<string, string>("B", "2")
};
var change = new CollectionDiff
{
DiffKind = CollectionDiffKind.FileNameChange,
Id = "B",
FileName = "newName"
};
OrderedCollectionDiffMerge.MergeChangeIntoOrderedItems(change, orderedItems);
Assert.AreEqual("newName", orderedItems[1].Value);
}
public void MergeChangeIntoOrderedItems_WithRemoveChange_RemovesTheItem()
{
var orderedItems = new List<KeyValuePair<string, string>>
{
new KeyValuePair<string, string>("A", "3"),
new KeyValuePair<string, string>("B", "2")
};
var change = new CollectionDiff() {DiffKind = CollectionDiffKind.Remove, Id = "A"};
OrderedCollectionDiffMerge.MergeChangeIntoOrderedItems(change, orderedItems);
Assert.AreEqual(-1, OrderedCollectionDiffMerge.GetItemIndex("A", orderedItems));
}
public void MergeChangeIntoOrderedItems_WithAddChange_AddsTheItem()
{
var orderedItems = new List<KeyValuePair<string, string>>
{
new KeyValuePair<string, string>("A", "3"),
new KeyValuePair<string, string>("B", "2")
};
var change = new CollectionDiff
{
DiffKind = CollectionDiffKind.Add,
Id = "C",
NowBeforeId = "A"
};
OrderedCollectionDiffMerge.MergeChangeIntoOrderedItems(change, orderedItems);
Assert.AreEqual(0, OrderedCollectionDiffMerge.GetItemIndex("C", orderedItems));
}
public async Task <SynchronizeIndexResponse> InternalExecute(SynchronizeIndexRequest request, CancellationToken cancellationToken = default)
{
var directory = request.Directory;
await using var dataContext = directory.GetDataContext();
var operations = new ConcurrentBag <FileOperation>();
State.Status = SynchronizeIndexStatus.Scanning;
// get all files from the repository
var indexedFiles = await dataContext.FileRepository.GetAllReadOnlyBySpecs(new IncludeFileLocationsSpec());
var indexedFileInfos = indexedFiles.SelectMany(x => x.ToFileInfos(directory));
// get all files from the actual directory
var localFiles = directory.EnumerateFiles().WithCancellation(cancellationToken).ToList();
State.Status = SynchronizeIndexStatus.Synchronizing;
// get changes
var(newFiles, removedFiles) = CollectionDiff.Create(indexedFileInfos, localFiles, new FileInfoComparer());
// files that are completely removed from the directory
var completelyRemovedFiles = new List <IFileInfo>();
// remove files from index
foreach (var removedFile in removedFiles)
{
var action = _serviceProvider.GetRequiredService <IRemoveFileFromIndexUseCase>();
var response = await action.Handle(new RemoveFileFromIndexRequest(removedFile.RelativeFilename !, directory));
if (action.HasError)
{
State.Errors.Add(removedFile.RelativeFilename !, action.Error !);
}
if (response !.IsCompletelyRemoved)
{
completelyRemovedFiles.Add(removedFile);
}
}
IImmutableList <FileInformation> removedFileInformation = removedFiles
.Select(x => GetFileInformationFromPath(x.RelativeFilename !, indexedFiles, directory))
.ToImmutableList();
var formerlyDeletedFiles = directory.MemoryManager.DirectoryMemory.DeletedFiles;
var deletedFilesLock = new object();
State.Status = SynchronizeIndexStatus.IndexingNewFiles;
State.TotalFiles = newFiles.Count;
var processedFilesCount = 0;
var removedFilesLock = new object();
var stateLock = new object();
await TaskCombinators.ThrottledCatchErrorsAsync(newFiles, async (newFile, _) =>
{
var(action, response) = await IndexFile(newFile.Filename, directory);
if (action.HasError)
{
lock (stateLock)
{
State.Errors.Add(newFile.Filename, action.Error !);
}
return;
}
var(indexedFile, fileLocation) = response !;
// remove from formerly deleted files
if (formerlyDeletedFiles.ContainsKey(indexedFile.Hash))
{
lock (deletedFilesLock)
{
formerlyDeletedFiles = formerlyDeletedFiles.Remove(indexedFile.Hash);
}
}
FileOperation fileOperation;
// get operation
var removedFile = removedFileInformation.FirstOrDefault(x => _fileContentComparer.Equals(x, indexedFile));
if (removedFile != null)
{
lock (removedFilesLock)
{
removedFileInformation = removedFileInformation.Remove(removedFile);
}
if (directory.PathComparer.Equals(fileLocation.RelativeFilename, removedFile.RelativeFilename !))
{
fileOperation = FileOperation.FileChanged(fileLocation, ToFileReference(removedFile));
}
else
{
fileOperation = FileOperation.FileMoved(fileLocation, ToFileReference(removedFile));
}
}
else
{
fileOperation = FileOperation.NewFile(fileLocation);
}
await using (var context = directory.GetDataContext())
{
await context.OperationRepository.Add(fileOperation);
operations.Add(fileOperation);
}
var processedFiles = Interlocked.Increment(ref processedFilesCount);
State.ProcessedFiles = processedFiles;
State.Progress = (double)processedFiles / newFiles.Count;
}, CancellationToken.None, 8); // do not use cancellation token here as a cancellation would destroy all move/change operations as all files were already removed
foreach (var removedFile in removedFileInformation)
{
var operation = FileOperation.FileRemoved(ToFileReference(removedFile));
await dataContext.OperationRepository.Add(operation);
operations.Add(operation);
// add the file to deleted files, if it was completely removed from index
// WARN: if a file changes, the previous file is not marked as deleted. Idk if that is actually desired
if (completelyRemovedFiles.Any(x => x.Filename == removedFile.Filename))
{
formerlyDeletedFiles = formerlyDeletedFiles.Add(removedFile.Hash.ToString(), new DeletedFileInfo(removedFile.RelativeFilename !, removedFile.Length, removedFile.Hash, removedFile.PhotoProperties, removedFile.FileCreatedOn, DateTimeOffset.UtcNow));
}
}
await directory.MemoryManager.Update(directory.MemoryManager.DirectoryMemory.SetDeletedFiles(formerlyDeletedFiles));
return(new SynchronizeIndexResponse(operations.ToList()));
}
