public void LinkThumbnail(ItemEx item, string thumbnailPath, out FileInfo file, out FileItemLinkChangeset linkCs, FileItemLink?linkSource = null)
{
FileHash hash = fileManagement.HashFile(thumbnailPath);
file = new FileInfoChangeset(item.DirectorySetups.FirstOrDefault(), new SysFileInfo(thumbnailPath), hash).ToEntity();
var existing = fileDataService.GetExtendedByHash().Lookup(file.HashKey.Value);
if (existing.HasValue)
{
file = existing.Value.FileInfo;
}
linkCs = new FileItemLinkChangeset(linkSource ?? item.LatestVersionedFile.Value.Link.Link)
{
ThumbnailKey = file.HashKey
};
this.fileDataService.Post(file);
}
public void CreateAndLinkThumbnail(ItemEx item, Action <FileStream> streamWriter, out FileInfo file, out FileItemLinkChangeset linkCs, FileItemLink?linkSource = null)
{
var relativeThumbnailPath = item.GenerateRelativeThumbnailPath();
var root = item.RootPath;
var absolutePath = Path.Combine(root, relativeThumbnailPath);
createThumbnailDir(absolutePath);
using (var fs = File.OpenWrite(absolutePath))
{
streamWriter(fs);
fs.Close();
}
LinkThumbnail(item, absolutePath, out file, out linkCs, linkSource);
}
public IEnumerable <ITaskProgressionInfo> CreateAndLinkThumbnail()
{
return(this.itemService
.GetExtended()
.Items
.Where(item =>
!item.LatestThumbnail.HasValue &&
item.LatestVersionedFile != null
)
.SplitEvenly(settingsService.ThreadCount)
.Select(chunk =>
{
TaskProgression prog = new TaskProgression();
prog.Title = "Creating Thumbnails";
prog.RequiredProgress = chunk.Count();
Observable.Start(() =>
{
prog.State = TaskState.InProgress;
var chunkResult = chunk.Select(item =>
{
FileInfo file = default;
FileItemLinkChangeset linkCs = null;
FailedThumbnailResult?err = null;
var imgPath = item.GenerateAbsoluteThumbnailPath();
var itemPath = item.LatestFilePath;
try
{
this.CreateAndLinkThumbnail(item, fs =>
{
createThumbnail(itemPath, fs);
}, out file, out linkCs);
}
catch (Exception ex)
{
err = new FailedThumbnailResult(ex, itemPath, imgPath);
prog.State = TaskState.RunningWithErrors;
}
prog.CurrentProgress++;
return new { file, linkCs, err };
}).ToArray();
var groupedResult = chunkResult.GroupBy(data => data.err.HasValue);
var successes = groupedResult.FirstOrDefault(x => !x.Key)?.Where(x => x != null);
var errors = groupedResult.FirstOrDefault(x => x.Key)?.Select(x => x.err.Value);
if (prog.State == TaskState.RunningWithErrors)
{
prog.State = successes?.Any() == true ? TaskState.PartialSuccess : TaskState.Failed;
}
else
{
prog.State = TaskState.Done;
}
if (errors?.Any() == true)
{
var eGroups = errors.GroupBy(e => e.Exception.GetType().FullName);
string msg = "the following Thumbnails could not be created";
foreach (var eGroup in eGroups)
{
msg += string.Join(Environment.NewLine, new string[] {
"",
"--------------------------------",
eGroup.Key,
"3dFile / thumbnail",
string.Join(Environment.NewLine, eGroup.Select(e => $"{e.FilePath} / {e.ThumbnailPath}"))
});
}
prog.Error = new Exception(msg);
}
}, RxApp.TaskpoolScheduler);
return prog as ITaskProgressionInfo;
})
.ToArray());
}
public void FullSync(IEnumerable <DirectorySetup> dirSetups)
{
Observable.Start(() =>
{
try
{
var job = new AsyncJobState("Full File Sync");
var mainProc = new AsyncProcessState("collecting data");
mainProc.AddProgress(3);
job.AddProcess(mainProc);
notificationCenter.Register(job);
mainProc.State = AsyncState.InProgress;
var localFiles = getLocalFiles(dirSetups);
var definedFiles = getDefinedFiles(dirSetups);
var paths = getDistinctPaths(localFiles, definedFiles);
var allFiles = (from file in paths
join localFile in localFiles
on file.ToLower() equals localFile.FileInfo.FullName.ToLower() into mLocalFile
join definedFile in definedFiles
on file.ToLower() equals definedFile.AbsolutePath.ToLower() into mDefinedFile
select new
{
localFile = mLocalFile.Any() ? (mLocalFile.First() as LocalFile?) : null,
definedFile = mDefinedFile.Any() ? mDefinedFile.First() as FileInfoEx? : null,
isNew = mLocalFile.Any() && !mDefinedFile.Any(),
isDeleted = !mLocalFile.Any() & mDefinedFile.Any(),
isUpdated = mLocalFile.Any() && mDefinedFile.Any() && mLocalFile.First().FileInfo.LastWriteTime != mDefinedFile.First().LastWriteTime || !mDefinedFile.FirstOrDefault().HashKey.HasValue,
unchanged = mLocalFile.Any() && mDefinedFile.Any() && mLocalFile.First().FileInfo.LastWriteTime == mDefinedFile.First().LastWriteTime
}).ToArray();
// stage 1: hash new Files
mainProc.OnNextStep("handling file updates");
var filesToHash = allFiles
.Where(x => x.isNew || x.isUpdated);
if (filesToHash.Any())
{
var updateProc = new AsyncProcessState("hashing files file updates");
updateProc.AddProgress(6);
job.AddProcess(updateProc);
var chunks = filesToHash
.SplitEvenly(settingsService.ThreadCount);
var hashProcs = chunks
.Select(chunk =>
{
Subject <IEnumerable <FilePatch> > result = new Subject <IEnumerable <FilePatch> >();
RxApp.TaskpoolScheduler.Schedule(() =>
{
var proc = new AsyncProcessState($"hashing #{chunk.Key}");
proc.AddProgress(chunk.Count());
lock (job)
job.AddProcess(proc);
proc.State = AsyncState.InProgress;
var hasher = SHA512.Create();
var res = chunk.Select(file =>
{
var res = new FilePatch(
file.definedFile.HasValue,
file.localFile.Value.DirectorySetup,
file.localFile.Value.FileInfo,
fileManagementService.HashFile(hasher, file.localFile.Value.FileInfo.FullName)
);
proc.OnNextStep();
return(res);
}).ToArray();
proc.State = AsyncState.Done;
result.OnNext(res);
});
return(result);
})
.ToArray();
hashProcs
.CombineLatest()
.Take(1)
.Subscribe(procRes =>
{
//var procRes = await hashProcs.CombineLatest();
var hashedFiles = procRes
.SelectMany(x => x)
.GroupBy(x => x.IsUpdate)
.ToArray();
updateProc.OnNextStep("preparing data");
var filesToUpdate = hashedFiles
.FirstOrDefault(x => x.Key == true)
?.Select(x => new { x.DirectorySetup, x.FileInfo, x.Hash, x.IsUpdate, HashKey = new FileInfoHashKey(x.Hash, x.FileInfo.Length) })
?.Join(
allFiles.Where(x => x.isUpdated),
x => x.FileInfo.FullName,
x => x.localFile?.FileInfo?.FullName,
(localFile, definedFile) => new
{
newHash = localFile.HashKey,
newFileInfo = localFile.FileInfo,
definedFile = definedFile.definedFile.Value
})
?.GroupBy(x => x.newHash);
var fileUpdateChangesets = new List <FileInfoChangeset>();
var fileItemLinkChangesets = new List <FileItemLinkChangeset>();
if (filesToUpdate != null)
{
updateProc.OnNextStep("creating linkupdates");
foreach (var files in filesToUpdate)
{
// prepare links
var relatedLinks = this.fileItemLinkService
.Get()
.KeyValues
.Where(x => files.Any(y => y.definedFile.HashKey == x.Value.FileKey || y.definedFile.HashKey == x.Value.FileKey));
foreach (var link in relatedLinks)
{
var versions = this.fileItemLinkService.Get().Items.Where(x => x.ItemId == link.Value.ItemId).Select(x => x.Version);
var version = GetNextVersion(link.Value.Version, versions);
var newLink = new FileItemLinkChangeset()
{
Version = version,
ThumbnailKey = link.Value.ThumbnailKey,
ItemId = link.Value.ItemId,
FileKey = link.Value.FileKey,
};
if (link.Value.FileKey == files.FirstOrDefault()?.definedFile.HashKey)
{
newLink.FileKey = files.FirstOrDefault().newHash;
}
if (link.Value.ThumbnailKey == files.FirstOrDefault()?.definedFile.HashKey)
{
newLink.ThumbnailKey = files.FirstOrDefault().newHash;
}
fileItemLinkChangesets.Add(newLink);
}
// prepare file
updateProc.OnNextStep("creating fileupdates");
fileUpdateChangesets.AddRange(
files.Select(file =>
{
var changeset = new FileInfoChangeset(file.definedFile.FileInfo);
changeset.SetSysFileInfo(file.definedFile.DirectorySetup, file.newFileInfo);
changeset.FileHash = file.newHash.FileHash;
changeset.DirectorySetupId = file.definedFile.DirectorySetup.Id;
changeset.Id = FileInfoId.New();
return(changeset);
}));
}
updateProc.OnNextStep("patching...");
this.fileDataService.Patch(fileUpdateChangesets);
this.fileItemLinkService.Patch(fileItemLinkChangesets);
}
else
{
updateProc.Skip(3, "no merge required");
}
updateProc.OnNextStep("handling new files");
var newFiles = hashedFiles
.FirstOrDefault(x => x.Key == false)
?.Select(file =>
{
var changeset = new FileInfoChangeset();
changeset.SetSysFileInfo(file.DirectorySetup, file.FileInfo);
changeset.FileHash = file.Hash;
changeset.DirectorySetupId = file.DirectorySetup.Id;
changeset.Id = FileInfoId.New();
return(changeset);
})
.ToArray();
if (newFiles != null)
{
this.fileDataService.Patch(newFiles);
}
updateProc.OnNextStep("done");
updateProc.State = AsyncState.Done;
},
(Exception ex) =>
{
updateProc.State = AsyncState.Failed;
updateProc.Title = "failed";
updateProc.Details = ex.ToString();
});
}
// stage 3: remove deleted Files
mainProc.OnNextStep("deleting old files");
this.fileDataService.Delete(allFiles.Where(x => x.isDeleted).Select(x => x.definedFile.Value.Id));
mainProc.OnNextStep("done");
mainProc.State = AsyncState.Done;
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), $"{nameof(LibraryManagementService)}.{nameof(FullSync)}");
}
}, RxApp.TaskpoolScheduler);
}
