/// <summary>
/// Applies a patch (content file) to the destination
/// </summary>
/// <param name="destination">The destination that contains the previous version of the data</param>
/// <param name="patch">The content file that the destination is patched with</param>
public void Patch(string[] destination, Library.Interface.ICompression patch)
{
Snapshots.ISystemIO SystemIO = Utility.Utility.IsClientLinux ? (Snapshots.ISystemIO)new Snapshots.SystemIOLinux() : (Snapshots.ISystemIO)new Snapshots.SystemIOWindows();
if (m_partialDeltas == null)
m_partialDeltas = new Dictionary<string, XervBackup.Library.Utility.TempFile>();
if (m_folderTimestamps == null)
m_folderTimestamps = new Dictionary<string, DateTime>();
for (int i = 0; i < destination.Length; i++)
destination[i] = Utility.Utility.AppendDirSeparator(destination[i]);
bool isUtc = patch.FileExists(UTC_TIME_MARKER);
//Set up the filter system to avoid dealing with filtered items
FilterHelper fh = new FilterHelper(this, destination, m_filter);
//Delete all files that were removed
if (patch.FileExists(DELETED_FILES))
foreach (string s in fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(DELETED_FILES)), false))
{
if (SystemIO.FileExists(s))
{
try
{
//TODO: Perhaps read ahead in patches to prevent creation
long size = SystemIO.FileLength(s);
SystemIO.FileDelete(s);
if (m_stat as RestoreStatistics != null)
{
(m_stat as RestoreStatistics).FilesRestored--;
(m_stat as RestoreStatistics).SizeOfRestoredFiles -= size;
(m_stat as RestoreStatistics).FilesDeleted++;
}
}
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogError(string.Format(Strings.RSyncDir.DeleteFileError, s, ex.Message), ex);
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.DeleteFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Warning, ex);
}
}
else
{
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.FileToDeleteMissingError, s), XervBackup.Library.Logging.LogMessageType.Warning);
}
}
//Delete all folders that were removed
if (patch.FileExists(DELETED_FOLDERS))
{
if (m_folders_to_delete == null)
m_folders_to_delete = new List<string>();
List<string> deletedfolders = new List<string>(fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(DELETED_FOLDERS)), true));
//Make sure subfolders are deleted first
deletedfolders.Sort();
deletedfolders.Reverse();
//Append to the list of folders to remove.
//The folders are removed when the patch sequence is finalized,
//because the deleted file list is not present until
//the last content file has been applied.
m_folders_to_delete.AddRange(deletedfolders);
}
//Add folders. This mainly applies to empty folders,
//as non-empty folders will also be created when files are restored
if (patch.FileExists(ADDED_FOLDERS))
{
List<string> addedfolders = new List<string>(fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(ADDED_FOLDERS)), true));
//Make sure topfolders are created first
addedfolders.Sort();
foreach (string s in addedfolders)
{
if (!SystemIO.DirectoryExists(s))
try
{
SystemIO.DirectoryCreate(s);
if (m_stat as RestoreStatistics != null)
(m_stat as RestoreStatistics).FoldersRestored++;
}
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogError(string.Format(Strings.RSyncDir.CreateFolderError, s, ex.Message), ex);
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.CreateFolderError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Warning, ex);
}
}
}
if (patch.FileExists(ADDED_FOLDERS_TIMESTAMPS))
{
//These times are always utc
string[] folders = FilenamesFromPlatformIndependant(patch.ReadAllLines(ADDED_FOLDERS));
string[] timestamps = patch.ReadAllLines(ADDED_FOLDERS_TIMESTAMPS);
for (int i = 0; i < folders.Length; i++)
m_folderTimestamps[RSyncDir.GetFullPathFromRelname(destination, folders[i])] = Utility.Utility.EPOCH.AddSeconds(long.Parse(timestamps[i]));
}
if (patch.FileExists(UPDATED_FOLDERS) && patch.FileExists(UPDATED_FOLDERS_TIMESTAMPS))
{
//These times are always utc
string[] folders = FilenamesFromPlatformIndependant(patch.ReadAllLines(UPDATED_FOLDERS));
string[] timestamps = patch.ReadAllLines(UPDATED_FOLDERS_TIMESTAMPS);
long l;
for (int i = 0; i < folders.Length; i++)
if (long.TryParse(timestamps[i], out l))
m_folderTimestamps[RSyncDir.GetFullPathFromRelname(destination, folders[i])] = Utility.Utility.EPOCH.AddSeconds(l);
}
PartialEntryRecord pe = null;
if (patch.FileExists(INCOMPLETE_FILE))
pe = new PartialEntryRecord(patch.ReadAllLines(INCOMPLETE_FILE));
PartialEntryRecord fe = null;
if (patch.FileExists(COMPLETED_FILE))
fe = new PartialEntryRecord(patch.ReadAllLines(COMPLETED_FILE));
int lastPg = -1;
string contentprefix = Utility.Utility.AppendDirSeparator(CONTENT_ROOT);
List<string> contentfiles = m_filter.FilterList(contentprefix, patch.ListFiles(contentprefix));
string deltaprefix = Utility.Utility.AppendDirSeparator(DELTA_ROOT);
List<string> deltafiles = m_filter.FilterList(deltaprefix, patch.ListFiles(deltaprefix));
string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT);
List<string> symlinks = m_filter.FilterList(symlinkprefix, patch.ListFiles(symlinkprefix));
long totalfiles = deltafiles.Count + contentfiles.Count;
long fileindex = 0;
//Restore new files
foreach (string s in contentfiles)
{
string target = GetFullPathFromRelname(destination, s.Substring(contentprefix.Length));
try
{
if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target)))
{
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderMissingError, target), XervBackup.Library.Logging.LogMessageType.Warning);
SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target));
}
//Update each 0.5%
int pg = (int)((fileindex / (double)totalfiles) * 200);
if (pg != lastPg)
{
ProgressEvent(pg / 2, target);
lastPg = pg;
}
using (System.IO.Stream s1 = patch.OpenRead(s))
{
PartialEntryRecord pex = null;
Utility.TempFile partialFile = null;
if (pe != null && string.Equals(pe.PlatformConvertedFilename, s))
pex = pe; //The file is incomplete
else if (fe != null && string.Equals(fe.PlatformConvertedFilename, s))
pex = fe; //The file has the final segment
if (pex != null && string.Equals(pex.PlatformConvertedFilename, s))
{
//Ensure that the partial file list is in the correct state
if (pex.StartOffset == 0 && m_partialDeltas.ContainsKey(s))
throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s));
else if (pex.StartOffset != 0 && !m_partialDeltas.ContainsKey(s))
throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s));
else if (pex.StartOffset == 0) //First entry, so create a temp file
m_partialDeltas.Add(s, new XervBackup.Library.Utility.TempFile());
partialFile = m_partialDeltas[s];
}
else if (m_partialDeltas.ContainsKey(s))
throw new Exception(string.Format(Strings.RSyncDir.FileShouldBePartialError, s));
long startOffset = pex == null ? 0 : pex.StartOffset;
using (System.IO.Stream s2 = SystemIO.FileOpenWrite(partialFile == null ? target : (string)partialFile))
{
if (s2.Length != startOffset)
throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s));
s2.Position = startOffset;
if (startOffset == 0)
s2.SetLength(0);
Utility.Utility.CopyStream(s1, s2);
}
if (pex != null && pex == fe)
{
if (SystemIO.FileExists(target))
SystemIO.FileDelete(target);
SystemIO.FileMove(partialFile, target);
partialFile.Dispose();
m_partialDeltas.Remove(s);
}
if (m_stat is RestoreStatistics && (partialFile == null || pex == fe))
{
(m_stat as RestoreStatistics).FilesRestored++;
(m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(target);
}
}
if (File.Exists(target))
{
DateTime t = patch.GetLastWriteTime(s);
if (!isUtc)
t = t.ToUniversalTime();
try { SystemIO.FileSetLastWriteTimeUtc(target, t); }
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.FailedToSetFileWriteTime, target, ex.Message), ex);
}
}
}
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex);
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex);
}
fileindex++;
}
//Patch modfied files
foreach (string s in deltafiles)
{
string target = GetFullPathFromRelname(destination, s.Substring(deltaprefix.Length));
try
{
//Update each 0.5%
int pg = (int)((fileindex / (double)totalfiles) * 200);
if (pg != lastPg)
{
ProgressEvent(pg / 2, target);
lastPg = pg;
}
if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target)))
{
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderDeltaError, target), XervBackup.Library.Logging.LogMessageType.Warning);
SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target));
}
PartialEntryRecord pex = null;
if (pe != null && string.Equals(pe.PlatformConvertedFilename, s))
pex = pe; //The file is incomplete
else if (fe != null && string.Equals(fe.PlatformConvertedFilename, s))
pex = fe; //The file has the final segment
Utility.TempFile tempDelta = null;
if (pex != null && string.Equals(pex.PlatformConvertedFilename, s))
{
//Ensure that the partial file list is in the correct state
if (pex.StartOffset == 0 && m_partialDeltas.ContainsKey(s))
throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s));
else if (pex.StartOffset != 0 && !m_partialDeltas.ContainsKey(s))
throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s));
else if (pex.StartOffset == 0) //First entry, so create a temp file
m_partialDeltas.Add(s, new XervBackup.Library.Utility.TempFile());
//Dump the content in the temp file at the specified offset
using (System.IO.Stream st = SystemIO.FileOpenWrite(m_partialDeltas[s]))
{
if (st.Length != pex.StartOffset)
throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s));
st.Position = pex.StartOffset;
using (System.IO.Stream s2 = patch.OpenRead(s))
Utility.Utility.CopyStream(s2, st);
}
//We can't process it until it is received completely
if (pex != fe)
continue;
tempDelta = m_partialDeltas[s];
m_partialDeltas.Remove(s);
}
else if (m_partialDeltas.ContainsKey(s))
throw new Exception(string.Format(Strings.RSyncDir.FileShouldBePartialError, s));
using (Utility.TempFile tempfile = new Utility.TempFile())
using (tempDelta) //May be null, but the using directive does not care
{
//Use either the patch directly, or the partial temp file
System.IO.Stream deltaStream = tempDelta == null ? patch.OpenRead(s) : SystemIO.FileOpenRead(tempDelta);
using (System.IO.Stream s2 = deltaStream)
using (System.IO.Stream s1 = SystemIO.FileOpenRead(target))
using (System.IO.Stream s3 = SystemIO.FileCreate(tempfile))
SharpRSync.Interface.PatchFile(s1, s2, s3);
if (m_stat as RestoreStatistics != null)
{
(m_stat as RestoreStatistics).SizeOfRestoredFiles -= SystemIO.FileLength(target);
(m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(tempfile);
(m_stat as RestoreStatistics).FilesPatched++;
}
SystemIO.FileDelete(target);
try { SystemIO.FileMove(tempfile, target); }
catch
{
//The OS sometimes reports the file as existing even after a delete
// this seems to be related to MS Security Essentials?
System.Threading.Thread.Sleep(500);
SystemIO.FileMove(tempfile, target);
}
}
if (File.Exists(target))
{
DateTime t = patch.GetLastWriteTime(s);
if (!isUtc)
t = t.ToUniversalTime();
try { SystemIO.FileSetLastWriteTimeUtc(target, t); }
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.FailedToSetFileWriteTime, target, ex.Message), ex);
}
}
}
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex);
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex);
try { SystemIO.FileDelete(target); }
catch { }
}
fileindex++;
}
//Re-create symlinks (no progress report here, should be really fast)
foreach (string s in symlinks)
{
string target = GetFullPathFromRelname(destination, s.Substring(symlinkprefix.Length));
string symlinktarget = "";
try
{
symlinktarget = FilenamesFromPlatformIndependant(new string[] { Encoding.UTF8.GetString(patch.ReadAllBytes(s)) })[0];
bool isDir = symlinktarget[symlinktarget.Length - 1] == Path.DirectorySeparatorChar;
if (isDir)
symlinktarget = symlinktarget.Substring(0, symlinktarget.Length - 1);
try
{
//In case another symlink is present, we "update" it
if (SystemIO.FileExists(target) && (SystemIO.GetFileAttributes(target) & FileAttributes.ReparsePoint) != 0)
SystemIO.FileDelete(target);
}
catch (Exception ex)
{
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex);
}
SystemIO.CreateSymlink(target, symlinktarget, isDir);
}
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex);
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), XervBackup.Library.Logging.LogMessageType.Error, ex);
try { SystemIO.FileDelete(target); }
catch { }
try
{
if (!string.IsNullOrEmpty(symlinktarget))
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(SystemIO.FileOpenWrite(target)))
sw.Write(symlinktarget);
}
catch
{
}
}
}
}
/// <summary>
/// Appends a file to the content and signature archives, watching the content archive file size.
/// Returns the partial file entry if the volume size was exceeded.
/// Returns null if the file was written entirely.
/// </summary>
/// <param name="entry">The entry that describes the partial file</param>
/// <param name="contentfile">The content archive file</param>
/// <param name="signaturefile">The signature archive file</param>
/// <param name="volumesize">The max allowed volumesize</param>
/// <returns>The partial file entry if the volume size was exceeded. Returns null if the file was written entirely.</returns>
private PartialFileEntry WritePossiblePartial(PartialFileEntry entry, Library.Interface.ICompression contentfile, Library.Interface.ICompression signaturefile, long volumesize)
{
long startPos = entry.Stream.Position;
//Protect against writing this file if there is not enough space to hold the INCOMPLETE_FILE
if (startPos == 0 && contentfile.Size + contentfile.FlushBufferSize + (entry.ExtraSize * 2) > volumesize)
return entry;
PartialFileEntry pe = WritePossiblePartialInternal(entry, contentfile, volumesize);
if (pe != null)
{
//The record is (still) partial
string[] tmplines = new PartialEntryRecord(entry.relativeName, startPos, entry.Stream.Position - startPos, entry.Stream.Length).Serialize();
contentfile.WriteAllLines(INCOMPLETE_FILE, tmplines);
signaturefile.WriteAllLines(INCOMPLETE_FILE, tmplines);
//If we are debugging, this can be nice to have
Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.PartialFileAddedLogMessage, entry.relativeName, startPos), XervBackup.Library.Logging.LogMessageType.Information);
}
else
{
//If the file was partial before, mark the file as completed
if (startPos != 0)
{
string[] tmplines = new PartialEntryRecord(entry.relativeName, startPos, entry.Stream.Position - startPos, entry.Stream.Length).Serialize();
contentfile.WriteAllLines(COMPLETED_FILE, tmplines);
signaturefile.WriteAllLines(COMPLETED_FILE, tmplines);
}
//Add signature AFTER content is completed.
//If content is present, it is restoreable, if signature is missing, file will be backed up on next run
//If signature is present, but not content, the entire differential sequence will be unable to recover the file
if (!entry.DumpSignature(signaturefile))
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.FileChangedWhileReadWarning, entry.fullname), null);
}
entry.Dispose();
}
return pe;
}
/// <summary>
/// Extracts the files found in a signature volume
/// </summary>
/// <param name="patchs">The signature volumes to read</param>
/// <returns>A list of file or folder names and their types</returns>
public List<KeyValuePair<PatchFileType, string>> ListPatchFiles(List<Library.Interface.ICompression> patches)
{
List<KeyValuePair<PatchFileType, string>> files = new List<KeyValuePair<PatchFileType, string>>();
KeyValuePair<PatchFileType, string>[] signatures = new KeyValuePair<PatchFileType, string>[] {
new KeyValuePair<PatchFileType, string>(PatchFileType.AddedOrUpdatedFile, Utility.Utility.AppendDirSeparator(COMBINED_SIGNATURE_ROOT)),
new KeyValuePair<PatchFileType, string>(PatchFileType.AddedFile, Utility.Utility.AppendDirSeparator(CONTENT_SIGNATURE_ROOT)),
new KeyValuePair<PatchFileType, string>(PatchFileType.UpdatedFile, Utility.Utility.AppendDirSeparator(DELTA_SIGNATURE_ROOT)),
};
string content_prefix = Utility.Utility.AppendDirSeparator(CONTENT_ROOT);
string delta_prefix = Utility.Utility.AppendDirSeparator(DELTA_ROOT);
string control_prefix = Utility.Utility.AppendDirSeparator(CONTROL_ROOT);
Dictionary<string, bool> partials = new Dictionary<string, bool>();
foreach (Library.Interface.ICompression arch in patches)
{
if (arch.FileExists(DELETED_FILES))
foreach (string s in FilenamesFromPlatformIndependant(arch.ReadAllLines(DELETED_FILES)))
files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.DeletedFile, s));
foreach(KeyValuePair<PatchFileType, string> sigentry in signatures)
foreach (string f in FilenamesFromPlatformIndependant(arch.ListFiles(sigentry.Value)))
{
if (partials.ContainsKey(f))
partials.Remove(f);
files.Add(new KeyValuePair<PatchFileType, string>(sigentry.Key, f.Substring(sigentry.Value.Length)));
}
foreach (string f in FilenamesFromPlatformIndependant(arch.ListFiles(control_prefix)))
files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.ControlFile, f.Substring(control_prefix.Length)));
if (arch.FileExists(DELETED_FOLDERS))
foreach (string s in FilenamesFromPlatformIndependant(arch.ReadAllLines(DELETED_FOLDERS)))
files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.DeletedFolder, s));
if (arch.FileExists(ADDED_FOLDERS))
foreach (string s in FilenamesFromPlatformIndependant(arch.ReadAllLines(ADDED_FOLDERS)))
files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.AddedFolder, s));
if (arch.FileExists(INCOMPLETE_FILE))
{
PartialEntryRecord pre = new PartialEntryRecord(arch.ReadAllLines(INCOMPLETE_FILE));
string filename = FilenamesFromPlatformIndependant(new string[] { pre.Filename })[0];
if (filename.StartsWith(content_prefix))
{
if (!partials.ContainsKey(filename.Substring(content_prefix.Length)))
partials.Add(filename.Substring(content_prefix.Length), false);
}
else if (filename.StartsWith(delta_prefix))
{
if (!partials.ContainsKey(filename.Substring(delta_prefix.Length)))
partials.Add(filename.Substring(delta_prefix.Length), false);
}
}
if (arch.FileExists(COMPLETED_FILE))
{
PartialEntryRecord pre = new PartialEntryRecord(arch.ReadAllLines(COMPLETED_FILE));
string filename = FilenamesFromPlatformIndependant(new string[] { pre.Filename })[0];
if (filename.StartsWith(content_prefix))
partials[filename.Substring(content_prefix.Length)]= true;
else if (filename.StartsWith(delta_prefix))
partials[filename.Substring(delta_prefix.Length)] = true;
}
}
foreach (KeyValuePair<string, bool> s in partials)
{
//Index of last found file that matches
int lastIx = -1;
for (int i = 0; i < files.Count; i++)
{
KeyValuePair<PatchFileType, string> px = files[i];
if ((px.Key == PatchFileType.AddedFile || px.Key == PatchFileType.AddedOrUpdatedFile || px.Key == PatchFileType.UpdatedFile) && px.Value == s.Key)
{
//We have a new file, if one is already found, remove it
if (lastIx != -1)
{
files.RemoveAt(lastIx);
i--;
}
lastIx = i;
}
}
//The file is incomplete, remove that only file entry, and insert the incomplete file entry
if (!s.Value)
{
if (lastIx != -1)
files.RemoveAt(lastIx);
files.Add(new KeyValuePair<PatchFileType, string>(PatchFileType.IncompleteFile, s.Key));
}
//If the file is completed, there is now only one entry left
}
return files;
}
