public void LinkThumbnail(ItemEx item, string thumbnailPath, out FileInfo file, out FileItemLinkChangeset linkCs, FileItemLink?linkSource = null)
{
FileHash hash = fileManagement.HashFile(thumbnailPath);
file = new FileInfoChangeset(item.DirectorySetups.FirstOrDefault(), new SysFileInfo(thumbnailPath), hash).ToEntity();
var existing = fileDataService.GetExtendedByHash().Lookup(file.HashKey.Value);
if (existing.HasValue)
{
file = existing.Value.FileInfo;
}
linkCs = new FileItemLinkChangeset(linkSource ?? item.LatestVersionedFile.Value.Link.Link)
{
ThumbnailKey = file.HashKey
};
this.fileDataService.Post(file);
}
public void FullSync(IEnumerable <DirectorySetup> dirSetups)
{
Observable.Start(() =>
{
try
{
var job = new AsyncJobState("Full File Sync");
var mainProc = new AsyncProcessState("collecting data");
mainProc.AddProgress(3);
job.AddProcess(mainProc);
notificationCenter.Register(job);
mainProc.State = AsyncState.InProgress;
var localFiles = getLocalFiles(dirSetups);
var definedFiles = getDefinedFiles(dirSetups);
var paths = getDistinctPaths(localFiles, definedFiles);
var allFiles = (from file in paths
join localFile in localFiles
on file.ToLower() equals localFile.FileInfo.FullName.ToLower() into mLocalFile
join definedFile in definedFiles
on file.ToLower() equals definedFile.AbsolutePath.ToLower() into mDefinedFile
select new
{
localFile = mLocalFile.Any() ? (mLocalFile.First() as LocalFile?) : null,
definedFile = mDefinedFile.Any() ? mDefinedFile.First() as FileInfoEx? : null,
isNew = mLocalFile.Any() && !mDefinedFile.Any(),
isDeleted = !mLocalFile.Any() & mDefinedFile.Any(),
isUpdated = mLocalFile.Any() && mDefinedFile.Any() && mLocalFile.First().FileInfo.LastWriteTime != mDefinedFile.First().LastWriteTime || !mDefinedFile.FirstOrDefault().HashKey.HasValue,
unchanged = mLocalFile.Any() && mDefinedFile.Any() && mLocalFile.First().FileInfo.LastWriteTime == mDefinedFile.First().LastWriteTime
}).ToArray();
// stage 1: hash new Files
mainProc.OnNextStep("handling file updates");
var filesToHash = allFiles
.Where(x => x.isNew || x.isUpdated);
if (filesToHash.Any())
{
var updateProc = new AsyncProcessState("hashing files file updates");
updateProc.AddProgress(6);
job.AddProcess(updateProc);
var chunks = filesToHash
.SplitEvenly(settingsService.ThreadCount);
var hashProcs = chunks
.Select(chunk =>
{
Subject <IEnumerable <FilePatch> > result = new Subject <IEnumerable <FilePatch> >();
RxApp.TaskpoolScheduler.Schedule(() =>
{
var proc = new AsyncProcessState($"hashing #{chunk.Key}");
proc.AddProgress(chunk.Count());
lock (job)
job.AddProcess(proc);
proc.State = AsyncState.InProgress;
var hasher = SHA512.Create();
var res = chunk.Select(file =>
{
var res = new FilePatch(
file.definedFile.HasValue,
file.localFile.Value.DirectorySetup,
file.localFile.Value.FileInfo,
fileManagementService.HashFile(hasher, file.localFile.Value.FileInfo.FullName)
);
proc.OnNextStep();
return(res);
}).ToArray();
proc.State = AsyncState.Done;
result.OnNext(res);
});
return(result);
})
.ToArray();
hashProcs
.CombineLatest()
.Take(1)
.Subscribe(procRes =>
{
//var procRes = await hashProcs.CombineLatest();
var hashedFiles = procRes
.SelectMany(x => x)
.GroupBy(x => x.IsUpdate)
.ToArray();
updateProc.OnNextStep("preparing data");
var filesToUpdate = hashedFiles
.FirstOrDefault(x => x.Key == true)
?.Select(x => new { x.DirectorySetup, x.FileInfo, x.Hash, x.IsUpdate, HashKey = new FileInfoHashKey(x.Hash, x.FileInfo.Length) })
?.Join(
allFiles.Where(x => x.isUpdated),
x => x.FileInfo.FullName,
x => x.localFile?.FileInfo?.FullName,
(localFile, definedFile) => new
{
newHash = localFile.HashKey,
newFileInfo = localFile.FileInfo,
definedFile = definedFile.definedFile.Value
})
?.GroupBy(x => x.newHash);
var fileUpdateChangesets = new List <FileInfoChangeset>();
var fileItemLinkChangesets = new List <FileItemLinkChangeset>();
if (filesToUpdate != null)
{
updateProc.OnNextStep("creating linkupdates");
foreach (var files in filesToUpdate)
{
// prepare links
var relatedLinks = this.fileItemLinkService
.Get()
.KeyValues
.Where(x => files.Any(y => y.definedFile.HashKey == x.Value.FileKey || y.definedFile.HashKey == x.Value.FileKey));
foreach (var link in relatedLinks)
{
var versions = this.fileItemLinkService.Get().Items.Where(x => x.ItemId == link.Value.ItemId).Select(x => x.Version);
var version = GetNextVersion(link.Value.Version, versions);
var newLink = new FileItemLinkChangeset()
{
Version = version,
ThumbnailKey = link.Value.ThumbnailKey,
ItemId = link.Value.ItemId,
FileKey = link.Value.FileKey,
};
if (link.Value.FileKey == files.FirstOrDefault()?.definedFile.HashKey)
{
newLink.FileKey = files.FirstOrDefault().newHash;
}
if (link.Value.ThumbnailKey == files.FirstOrDefault()?.definedFile.HashKey)
{
newLink.ThumbnailKey = files.FirstOrDefault().newHash;
}
fileItemLinkChangesets.Add(newLink);
}
// prepare file
updateProc.OnNextStep("creating fileupdates");
fileUpdateChangesets.AddRange(
files.Select(file =>
{
var changeset = new FileInfoChangeset(file.definedFile.FileInfo);
changeset.SetSysFileInfo(file.definedFile.DirectorySetup, file.newFileInfo);
changeset.FileHash = file.newHash.FileHash;
changeset.DirectorySetupId = file.definedFile.DirectorySetup.Id;
changeset.Id = FileInfoId.New();
return(changeset);
}));
}
updateProc.OnNextStep("patching...");
this.fileDataService.Patch(fileUpdateChangesets);
this.fileItemLinkService.Patch(fileItemLinkChangesets);
}
else
{
updateProc.Skip(3, "no merge required");
}
updateProc.OnNextStep("handling new files");
var newFiles = hashedFiles
.FirstOrDefault(x => x.Key == false)
?.Select(file =>
{
var changeset = new FileInfoChangeset();
changeset.SetSysFileInfo(file.DirectorySetup, file.FileInfo);
changeset.FileHash = file.Hash;
changeset.DirectorySetupId = file.DirectorySetup.Id;
changeset.Id = FileInfoId.New();
return(changeset);
})
.ToArray();
if (newFiles != null)
{
this.fileDataService.Patch(newFiles);
}
updateProc.OnNextStep("done");
updateProc.State = AsyncState.Done;
},
(Exception ex) =>
{
updateProc.State = AsyncState.Failed;
updateProc.Title = "failed";
updateProc.Details = ex.ToString();
});
}
// stage 3: remove deleted Files
mainProc.OnNextStep("deleting old files");
this.fileDataService.Delete(allFiles.Where(x => x.isDeleted).Select(x => x.definedFile.Value.Id));
mainProc.OnNextStep("done");
mainProc.State = AsyncState.Done;
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), $"{nameof(LibraryManagementService)}.{nameof(FullSync)}");
}
}, RxApp.TaskpoolScheduler);
}
