/// <summary> /// Applies a patch (content file) to the destination /// </summary> /// <param name="destination">The destination that contains the previous version of the data</param> /// <param name="patch">The content file that the destination is patched with</param> public void Patch(string[] destination, Library.Interface.ICompression patch) { Snapshots.ISystemIO SystemIO = Utility.Utility.IsClientLinux ? (Snapshots.ISystemIO)new Snapshots.SystemIOLinux() : (Snapshots.ISystemIO)new Snapshots.SystemIOWindows(); if (m_partialDeltas == null) m_partialDeltas = new Dictionary<string, XervBackup.Library.Utility.TempFile>(); if (m_folderTimestamps == null) m_folderTimestamps = new Dictionary<string, DateTime>(); for (int i = 0; i < destination.Length; i++) destination[i] = Utility.Utility.AppendDirSeparator(destination[i]); bool isUtc = patch.FileExists(UTC_TIME_MARKER); //Set up the filter system to avoid dealing with filtered items FilterHelper fh = new FilterHelper(this, destination, m_filter); //Delete all files that were removed if (patch.FileExists(DELETED_FILES)) foreach (string s in fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(DELETED_FILES)), false)) { if (SystemIO.FileExists(s)) { try { //TODO: Perhaps read ahead in patches to prevent creation long size = SystemIO.FileLength(s); SystemIO.FileDelete(s); if (m_stat as RestoreStatistics != null) { (m_stat as RestoreStatistics).FilesRestored--; (m_stat as RestoreStatistics).SizeOfRestoredFiles -= size; (m_stat as RestoreStatistics).FilesDeleted++; } } catch (Exception ex) { if (m_stat != null) m_stat.LogError(string.Format(Strings.RSyncDir.DeleteFileError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.DeleteFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Warning, ex); } } else { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.FileToDeleteMissingError, s), XervBackup.Library.Logging.LogMessageType.Warning); } } //Delete all folders that were removed if (patch.FileExists(DELETED_FOLDERS)) { if (m_folders_to_delete == null) m_folders_to_delete = new List<string>(); List<string> deletedfolders = new List<string>(fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(DELETED_FOLDERS)), true)); //Make sure subfolders are deleted first deletedfolders.Sort(); deletedfolders.Reverse(); //Append to the list of folders to remove. //The folders are removed when the patch sequence is finalized, //because the deleted file list is not present until //the last content file has been applied. m_folders_to_delete.AddRange(deletedfolders); } //Add folders. This mainly applies to empty folders, //as non-empty folders will also be created when files are restored if (patch.FileExists(ADDED_FOLDERS)) { List<string> addedfolders = new List<string>(fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(ADDED_FOLDERS)), true)); //Make sure topfolders are created first addedfolders.Sort(); foreach (string s in addedfolders) { if (!SystemIO.DirectoryExists(s)) try { SystemIO.DirectoryCreate(s); if (m_stat as RestoreStatistics != null) (m_stat as RestoreStatistics).FoldersRestored++; } catch (Exception ex) { if (m_stat != null) m_stat.LogError(string.Format(Strings.RSyncDir.CreateFolderError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.CreateFolderError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Warning, ex); } } } if (patch.FileExists(ADDED_FOLDERS_TIMESTAMPS)) { //These times are always utc string[] folders = FilenamesFromPlatformIndependant(patch.ReadAllLines(ADDED_FOLDERS)); string[] timestamps = patch.ReadAllLines(ADDED_FOLDERS_TIMESTAMPS); for (int i = 0; i < folders.Length; i++) m_folderTimestamps[RSyncDir.GetFullPathFromRelname(destination, folders[i])] = Utility.Utility.EPOCH.AddSeconds(long.Parse(timestamps[i])); } if (patch.FileExists(UPDATED_FOLDERS) && patch.FileExists(UPDATED_FOLDERS_TIMESTAMPS)) { //These times are always utc string[] folders = FilenamesFromPlatformIndependant(patch.ReadAllLines(UPDATED_FOLDERS)); string[] timestamps = patch.ReadAllLines(UPDATED_FOLDERS_TIMESTAMPS); long l; for (int i = 0; i < folders.Length; i++) if (long.TryParse(timestamps[i], out l)) m_folderTimestamps[RSyncDir.GetFullPathFromRelname(destination, folders[i])] = Utility.Utility.EPOCH.AddSeconds(l); } PartialEntryRecord pe = null; if (patch.FileExists(INCOMPLETE_FILE)) pe = new PartialEntryRecord(patch.ReadAllLines(INCOMPLETE_FILE)); PartialEntryRecord fe = null; if (patch.FileExists(COMPLETED_FILE)) fe = new PartialEntryRecord(patch.ReadAllLines(COMPLETED_FILE)); int lastPg = -1; string contentprefix = Utility.Utility.AppendDirSeparator(CONTENT_ROOT); List<string> contentfiles = m_filter.FilterList(contentprefix, patch.ListFiles(contentprefix)); string deltaprefix = Utility.Utility.AppendDirSeparator(DELTA_ROOT); List<string> deltafiles = m_filter.FilterList(deltaprefix, patch.ListFiles(deltaprefix)); string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT); List<string> symlinks = m_filter.FilterList(symlinkprefix, patch.ListFiles(symlinkprefix)); long totalfiles = deltafiles.Count + contentfiles.Count; long fileindex = 0; //Restore new files foreach (string s in contentfiles) { string target = GetFullPathFromRelname(destination, s.Substring(contentprefix.Length)); try { if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target))) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderMissingError, target), XervBackup.Library.Logging.LogMessageType.Warning); SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target)); } //Update each 0.5% int pg = (int)((fileindex / (double)totalfiles) * 200); if (pg != lastPg) { ProgressEvent(pg / 2, target); lastPg = pg; } using (System.IO.Stream s1 = patch.OpenRead(s)) { PartialEntryRecord pex = null; Utility.TempFile partialFile = null; if (pe != null && string.Equals(pe.PlatformConvertedFilename, s)) pex = pe; //The file is incomplete else if (fe != null && string.Equals(fe.PlatformConvertedFilename, s)) pex = fe; //The file has the final segment if (pex != null && string.Equals(pex.PlatformConvertedFilename, s)) { //Ensure that the partial file list is in the correct state if (pex.StartOffset == 0 && m_partialDeltas.ContainsKey(s)) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); else if (pex.StartOffset != 0 && !m_partialDeltas.ContainsKey(s)) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); else if (pex.StartOffset == 0) //First entry, so create a temp file m_partialDeltas.Add(s, new XervBackup.Library.Utility.TempFile()); partialFile = m_partialDeltas[s]; } else if (m_partialDeltas.ContainsKey(s)) throw new Exception(string.Format(Strings.RSyncDir.FileShouldBePartialError, s)); long startOffset = pex == null ? 0 : pex.StartOffset; using (System.IO.Stream s2 = SystemIO.FileOpenWrite(partialFile == null ? target : (string)partialFile)) { if (s2.Length != startOffset) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); s2.Position = startOffset; if (startOffset == 0) s2.SetLength(0); Utility.Utility.CopyStream(s1, s2); } if (pex != null && pex == fe) { if (SystemIO.FileExists(target)) SystemIO.FileDelete(target); SystemIO.FileMove(partialFile, target); partialFile.Dispose(); m_partialDeltas.Remove(s); } if (m_stat is RestoreStatistics && (partialFile == null || pex == fe)) { (m_stat as RestoreStatistics).FilesRestored++; (m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(target); } } if (File.Exists(target)) { DateTime t = patch.GetLastWriteTime(s); if (!isUtc) t = t.ToUniversalTime(); try { SystemIO.FileSetLastWriteTimeUtc(target, t); } catch (Exception ex) { if (m_stat != null) m_stat.LogWarning(string.Format(Strings.RSyncDir.FailedToSetFileWriteTime, target, ex.Message), ex); } } } catch (Exception ex) { if (m_stat != null) m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex); } fileindex++; } //Patch modfied files foreach (string s in deltafiles) { string target = GetFullPathFromRelname(destination, s.Substring(deltaprefix.Length)); try { //Update each 0.5% int pg = (int)((fileindex / (double)totalfiles) * 200); if (pg != lastPg) { ProgressEvent(pg / 2, target); lastPg = pg; } if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target))) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderDeltaError, target), XervBackup.Library.Logging.LogMessageType.Warning); SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target)); } PartialEntryRecord pex = null; if (pe != null && string.Equals(pe.PlatformConvertedFilename, s)) pex = pe; //The file is incomplete else if (fe != null && string.Equals(fe.PlatformConvertedFilename, s)) pex = fe; //The file has the final segment Utility.TempFile tempDelta = null; if (pex != null && string.Equals(pex.PlatformConvertedFilename, s)) { //Ensure that the partial file list is in the correct state if (pex.StartOffset == 0 && m_partialDeltas.ContainsKey(s)) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); else if (pex.StartOffset != 0 && !m_partialDeltas.ContainsKey(s)) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); else if (pex.StartOffset == 0) //First entry, so create a temp file m_partialDeltas.Add(s, new XervBackup.Library.Utility.TempFile()); //Dump the content in the temp file at the specified offset using (System.IO.Stream st = SystemIO.FileOpenWrite(m_partialDeltas[s])) { if (st.Length != pex.StartOffset) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); st.Position = pex.StartOffset; using (System.IO.Stream s2 = patch.OpenRead(s)) Utility.Utility.CopyStream(s2, st); } //We can't process it until it is received completely if (pex != fe) continue; tempDelta = m_partialDeltas[s]; m_partialDeltas.Remove(s); } else if (m_partialDeltas.ContainsKey(s)) throw new Exception(string.Format(Strings.RSyncDir.FileShouldBePartialError, s)); using (Utility.TempFile tempfile = new Utility.TempFile()) using (tempDelta) //May be null, but the using directive does not care { //Use either the patch directly, or the partial temp file System.IO.Stream deltaStream = tempDelta == null ? patch.OpenRead(s) : SystemIO.FileOpenRead(tempDelta); using (System.IO.Stream s2 = deltaStream) using (System.IO.Stream s1 = SystemIO.FileOpenRead(target)) using (System.IO.Stream s3 = SystemIO.FileCreate(tempfile)) SharpRSync.Interface.PatchFile(s1, s2, s3); if (m_stat as RestoreStatistics != null) { (m_stat as RestoreStatistics).SizeOfRestoredFiles -= SystemIO.FileLength(target); (m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(tempfile); (m_stat as RestoreStatistics).FilesPatched++; } SystemIO.FileDelete(target); try { SystemIO.FileMove(tempfile, target); } catch { //The OS sometimes reports the file as existing even after a delete // this seems to be related to MS Security Essentials? System.Threading.Thread.Sleep(500); SystemIO.FileMove(tempfile, target); } } if (File.Exists(target)) { DateTime t = patch.GetLastWriteTime(s); if (!isUtc) t = t.ToUniversalTime(); try { SystemIO.FileSetLastWriteTimeUtc(target, t); } catch (Exception ex) { if (m_stat != null) m_stat.LogWarning(string.Format(Strings.RSyncDir.FailedToSetFileWriteTime, target, ex.Message), ex); } } } catch (Exception ex) { if (m_stat != null) m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex); try { SystemIO.FileDelete(target); } catch { } } fileindex++; } //Re-create symlinks (no progress report here, should be really fast) foreach (string s in symlinks) { string target = GetFullPathFromRelname(destination, s.Substring(symlinkprefix.Length)); string symlinktarget = ""; try { symlinktarget = FilenamesFromPlatformIndependant(new string[] { Encoding.UTF8.GetString(patch.ReadAllBytes(s)) })[0]; bool isDir = symlinktarget[symlinktarget.Length - 1] == Path.DirectorySeparatorChar; if (isDir) symlinktarget = symlinktarget.Substring(0, symlinktarget.Length - 1); try { //In case another symlink is present, we "update" it if (SystemIO.FileExists(target) && (SystemIO.GetFileAttributes(target) & FileAttributes.ReparsePoint) != 0) SystemIO.FileDelete(target); } catch (Exception ex) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex); } SystemIO.CreateSymlink(target, symlinktarget, isDir); } catch (Exception ex) { if (m_stat != null) m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex); try { SystemIO.FileDelete(target); } catch { } try { if (!string.IsNullOrEmpty(symlinktarget)) using (System.IO.StreamWriter sw = new System.IO.StreamWriter(SystemIO.FileOpenWrite(target))) sw.Write(symlinktarget); } catch { } } } }
/// <summary> /// Appends a file to the content and signature archives, watching the content archive file size. /// Returns the partial file entry if the volume size was exceeded. /// Returns null if the file was written entirely. /// </summary> /// <param name="entry">The entry that describes the partial file</param> /// <param name="contentfile">The content archive file</param> /// <param name="signaturefile">The signature archive file</param> /// <param name="volumesize">The max allowed volumesize</param> /// <returns>The partial file entry if the volume size was exceeded. Returns null if the file was written entirely.</returns> private PartialFileEntry WritePossiblePartial(PartialFileEntry entry, Library.Interface.ICompression contentfile, Library.Interface.ICompression signaturefile, long volumesize) { long startPos = entry.Stream.Position; //Protect against writing this file if there is not enough space to hold the INCOMPLETE_FILE if (startPos == 0 && contentfile.Size + contentfile.FlushBufferSize + (entry.ExtraSize * 2) > volumesize) return entry; PartialFileEntry pe = WritePossiblePartialInternal(entry, contentfile, volumesize); if (pe != null) { //The record is (still) partial string[] tmplines = new PartialEntryRecord(entry.relativeName, startPos, entry.Stream.Position - startPos, entry.Stream.Length).Serialize(); contentfile.WriteAllLines(INCOMPLETE_FILE, tmplines); signaturefile.WriteAllLines(INCOMPLETE_FILE, tmplines); //If we are debugging, this can be nice to have Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.PartialFileAddedLogMessage, entry.relativeName, startPos), XervBackup.Library.Logging.LogMessageType.Information); } else { //If the file was partial before, mark the file as completed if (startPos != 0) { string[] tmplines = new PartialEntryRecord(entry.relativeName, startPos, entry.Stream.Position - startPos, entry.Stream.Length).Serialize(); contentfile.WriteAllLines(COMPLETED_FILE, tmplines); signaturefile.WriteAllLines(COMPLETED_FILE, tmplines); } //Add signature AFTER content is completed. //If content is present, it is restoreable, if signature is missing, file will be backed up on next run //If signature is present, but not content, the entire differential sequence will be unable to recover the file if (!entry.DumpSignature(signaturefile)) { if (m_stat != null) m_stat.LogWarning(string.Format(Strings.RSyncDir.FileChangedWhileReadWarning, entry.fullname), null); } entry.Dispose(); } return pe; }
/// <summary> /// Extracts the files found in a signature volume /// </summary> /// <param name="patchs">The signature volumes to read</param> /// <returns>A list of file or folder names and their types</returns> public List<KeyValuePair<PatchFileType, string>> ListPatchFiles(List<Library.Interface.ICompression> patches) { List<KeyValuePair<PatchFileType, string>> files = new List<KeyValuePair<PatchFileType, string>>(); KeyValuePair<PatchFileType, string>[] signatures = new KeyValuePair<PatchFileType, string>[] { new KeyValuePair<PatchFileType, string>(PatchFileType.AddedOrUpdatedFile, Utility.Utility.AppendDirSeparator(COMBINED_SIGNATURE_ROOT)), new KeyValuePair<PatchFileType, string>(PatchFileType.AddedFile, Utility.Utility.AppendDirSeparator(CONTENT_SIGNATURE_ROOT)), new KeyValuePair<PatchFileType, string>(PatchFileType.UpdatedFile, Utility.Utility.AppendDirSeparator(DELTA_SIGNATURE_ROOT)), }; string content_prefix = Utility.Utility.AppendDirSeparator(CONTENT_ROOT); string delta_prefix = Utility.Utility.AppendDirSeparator(DELTA_ROOT); string control_prefix = Utility.Utility.AppendDirSeparator(CONTROL_ROOT); Dictionary<string, bool> partials = new Dictionary<string, bool>(); foreach (Library.Interface.ICompression arch in patches) { if (arch.FileExists(DELETED_FILES)) foreach (string s in FilenamesFromPlatformIndependant(arch.ReadAllLines(DELETED_FILES))) files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.DeletedFile, s)); foreach(KeyValuePair<PatchFileType, string> sigentry in signatures) foreach (string f in FilenamesFromPlatformIndependant(arch.ListFiles(sigentry.Value))) { if (partials.ContainsKey(f)) partials.Remove(f); files.Add(new KeyValuePair<PatchFileType, string>(sigentry.Key, f.Substring(sigentry.Value.Length))); } foreach (string f in FilenamesFromPlatformIndependant(arch.ListFiles(control_prefix))) files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.ControlFile, f.Substring(control_prefix.Length))); if (arch.FileExists(DELETED_FOLDERS)) foreach (string s in FilenamesFromPlatformIndependant(arch.ReadAllLines(DELETED_FOLDERS))) files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.DeletedFolder, s)); if (arch.FileExists(ADDED_FOLDERS)) foreach (string s in FilenamesFromPlatformIndependant(arch.ReadAllLines(ADDED_FOLDERS))) files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.AddedFolder, s)); if (arch.FileExists(INCOMPLETE_FILE)) { PartialEntryRecord pre = new PartialEntryRecord(arch.ReadAllLines(INCOMPLETE_FILE)); string filename = FilenamesFromPlatformIndependant(new string[] { pre.Filename })[0]; if (filename.StartsWith(content_prefix)) { if (!partials.ContainsKey(filename.Substring(content_prefix.Length))) partials.Add(filename.Substring(content_prefix.Length), false); } else if (filename.StartsWith(delta_prefix)) { if (!partials.ContainsKey(filename.Substring(delta_prefix.Length))) partials.Add(filename.Substring(delta_prefix.Length), false); } } if (arch.FileExists(COMPLETED_FILE)) { PartialEntryRecord pre = new PartialEntryRecord(arch.ReadAllLines(COMPLETED_FILE)); string filename = FilenamesFromPlatformIndependant(new string[] { pre.Filename })[0]; if (filename.StartsWith(content_prefix)) partials[filename.Substring(content_prefix.Length)]= true; else if (filename.StartsWith(delta_prefix)) partials[filename.Substring(delta_prefix.Length)] = true; } } foreach (KeyValuePair<string, bool> s in partials) { //Index of last found file that matches int lastIx = -1; for (int i = 0; i < files.Count; i++) { KeyValuePair<PatchFileType, string> px = files[i]; if ((px.Key == PatchFileType.AddedFile || px.Key == PatchFileType.AddedOrUpdatedFile || px.Key == PatchFileType.UpdatedFile) && px.Value == s.Key) { //We have a new file, if one is already found, remove it if (lastIx != -1) { files.RemoveAt(lastIx); i--; } lastIx = i; } } //The file is incomplete, remove that only file entry, and insert the incomplete file entry if (!s.Value) { if (lastIx != -1) files.RemoveAt(lastIx); files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.IncompleteFile, s.Key)); } //If the file is completed, there is now only one entry left } return files; }