/// <summary>
/// Initializes a RSyncDir instance, binds it to the source folder, and reads in the supplied patches
/// </summary>
/// <param name="sourcefolder">The folders to create a backup from</param>
/// <param name="stat">The status report object</param>
/// <param name="filter">An optional filter that controls what files to include</param>
/// <param name="patches">A list of signature archives to read, MUST be sorted in the creation order, oldest first</param>
public RSyncDir(string[] sourcefolder, CommunicationStatistics stat, Utility.FilenameFilter filter, List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches)
: this(sourcefolder, stat, filter)
{
string[] prefixes = new string[] {
Utility.Utility.AppendDirSeparator(COMBINED_SIGNATURE_ROOT),
Utility.Utility.AppendDirSeparator(CONTENT_SIGNATURE_ROOT),
Utility.Utility.AppendDirSeparator(DELTA_SIGNATURE_ROOT)
};
m_patches = new List<XervBackup.Library.Interface.ICompression>();
foreach (KeyValuePair<ManifestEntry, Library.Interface.ICompression> patch in patches)
m_patches.Add(patch.Value);
foreach (KeyValuePair<ManifestEntry, Library.Interface.ICompression> patch in patches)
{
Library.Interface.ICompression z = patch.Value;
if (z.FileExists(DELETED_FILES))
foreach (string s in FilenamesFromPlatformIndependant(z.ReadAllLines(DELETED_FILES)))
{
m_oldSignatures.Remove(s);
m_lastVerificationTime.Remove(s);
m_oldSymlinks.Remove(s);
}
foreach (string prefix in prefixes)
{
ArchiveWrapper aw = new ArchiveWrapper(z, patch.Key.Time.ToUniversalTime(), prefix);
foreach (string f in FilenamesFromPlatformIndependant(z.ListFiles(prefix)))
{
string name = f.Substring(prefix.Length);
m_oldSignatures[name] = aw;
m_lastVerificationTime.Remove(name);
}
}
string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT);
foreach(string s in FilenamesFromPlatformIndependant(patch.Value.ListFiles(symlinkprefix)))
{
string tmp = FilenamesFromPlatformIndependant( new string[] { Encoding.UTF8.GetString(patch.Value.ReadAllBytes(s)) })[0];
m_oldSymlinks[s.Substring(symlinkprefix.Length)] = tmp;
}
if (z.FileExists(UNMODIFIED_FILES))
foreach (string s in FilenamesFromPlatformIndependant(z.ReadAllLines(UNMODIFIED_FILES)))
m_lastVerificationTime[s] = patch.Key.Time.ToUniversalTime();
if (z.FileExists(DELETED_FOLDERS))
foreach (string s in FilenamesFromPlatformIndependant(z.ReadAllLines(DELETED_FOLDERS)))
m_oldFolders.Remove(s);
if (z.FileExists(ADDED_FOLDERS))
{
DateTime t = z.GetLastWriteTime(ADDED_FOLDERS).ToUniversalTime();
string[] filenames = FilenamesFromPlatformIndependant(z.ReadAllLines(ADDED_FOLDERS));
if (z.FileExists(ADDED_FOLDERS_TIMESTAMPS))
{
string[] timestamps = z.ReadAllLines(ADDED_FOLDERS_TIMESTAMPS);
long l;
for(int i = 0; i < Math.Min(filenames.Length, timestamps.Length); i++)
if (long.TryParse(timestamps[i], out l))
m_oldFolders[filenames[i]] = Library.Utility.Utility.EPOCH.AddSeconds(l);
else
m_oldFolders[filenames[i]] = t;
}
else
{
foreach (string s in filenames)
m_oldFolders[s] = t;
}
}
if (z.FileExists(UPDATED_FOLDERS) && z.FileExists(UPDATED_FOLDERS_TIMESTAMPS))
{
string[] filenames = FilenamesFromPlatformIndependant(z.ReadAllLines(UPDATED_FOLDERS));
string[] timestamps = z.ReadAllLines(UPDATED_FOLDERS_TIMESTAMPS);
long l;
for (int i = 0; i < Math.Min(filenames.Length, timestamps.Length); i++)
if (long.TryParse(timestamps[i], out l))
m_oldFolders[filenames[i]] = Utility.Utility.EPOCH.AddSeconds(l);
}
//The latest file is the most valid
if (z.FileExists(USN_VALUES))
using (System.IO.Stream s = z.OpenRead(USN_VALUES))
m_lastUSN = new USNRecord(s);
}
}
/// <summary>
/// Initiates creating a content/signature pair.
/// </summary>
/// <param name="full">True if the set is a full backup, false if it is incremental</param>
/// <param name="options">Any setup options to use</param>
public void InitiateMultiPassDiff(bool full, Options options)
{
if (full)
{
m_oldFolders = new Dictionary<string, DateTime>();
m_oldSignatures = new Dictionary<string, ArchiveWrapper>();
m_oldSymlinks = new Dictionary<string, string>();
}
m_newfiles = new Dictionary<string, string>();
m_modifiedFiles = new Dictionary<string, string>();
m_deletedfiles = new List<string>();
m_newfolders = new List<KeyValuePair<string, DateTime>>();
m_updatedfolders = new List<KeyValuePair<string, DateTime>>();
m_deletedfolders = new List<string>();
m_checkedUnchangedFiles = new List<string>();
m_lastPartialFile = null;
m_openfilepolicy = Options.OpenFileStrategy.Ignore;
try
{
if (options.SnapShotStrategy != Options.OptimizationStrategy.Off)
m_snapshot = XervBackup.Library.Snapshots.SnapshotUtility.CreateSnapshot(m_sourcefolder, options.RawOptions);
}
catch (Exception ex)
{
if (options.SnapShotStrategy == Options.OptimizationStrategy.Required)
throw;
else if (options.SnapShotStrategy == Options.OptimizationStrategy.On)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.SnapshotFailedError, ex.ToString()), ex);
}
}
//Failsafe, just use a plain implementation
if (m_snapshot == null)
{
m_snapshot = Utility.Utility.IsClientLinux ?
(Library.Snapshots.ISnapshotService)new XervBackup.Library.Snapshots.NoSnapshotLinux(m_sourcefolder, options.RawOptions)
:
(Library.Snapshots.ISnapshotService)new XervBackup.Library.Snapshots.NoSnapshotWindows(m_sourcefolder, options.RawOptions);
m_openfilepolicy = options.OpenFilePolicy;
}
Dictionary<string, Snapshots.USNHelper> usnHelpers = null;
List<string> unchanged = new List<string>();
m_unproccesed = new PathCollector(m_snapshot, options.SymlinkPolicy, options.FileAttributeFilter, m_filter, m_stat);
try
{
if (options.UsnStrategy != Options.OptimizationStrategy.Off)
{
if (Utility.Utility.IsClientLinux && options.UsnStrategy != Options.OptimizationStrategy.Auto)
throw new Exception(Strings.RSyncDir.UsnNotSupportedOnLinuxError);
/*
if (options.DisableUSNDiffCheck)
m_lastUSN = null;
*/
usnHelpers = new Dictionary<string, XervBackup.Library.Snapshots.USNHelper>(Utility.Utility.ClientFilenameStringComparer);
foreach (string s in m_sourcefolder)
{
string rootFolder = System.IO.Path.GetPathRoot(s);
if (!usnHelpers.ContainsKey(rootFolder))
try { usnHelpers[rootFolder] = new XervBackup.Library.Snapshots.USNHelper(rootFolder); }
catch (Exception ex)
{
if (options.UsnStrategy == Options.OptimizationStrategy.Required)
throw;
else if (options.UsnStrategy == Options.OptimizationStrategy.On)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.UsnFailedError, ex.ToString()), ex);
}
}
if (usnHelpers.ContainsKey(rootFolder))
{
//This code is broken, see issue 332:
//http://code.google.com/p/XervBackup/issues/detail?id=332
/* if (m_lastUSN != null && m_lastUSN.ContainsKey(rootFolder))
{
if (m_lastUSN[rootFolder].Key != usnHelpers[rootFolder].JournalID)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.UsnJournalIdChangedWarning, rootFolder, m_lastUSN[rootFolder].Key, usnHelpers[rootFolder].JournalID), null);
//Just take it all again
usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_filter, m_unproccesed.Callback);
}
else if (m_lastUSN[rootFolder].Value > usnHelpers[rootFolder].USN)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.UsnNumberingFaultWarning, rootFolder, m_lastUSN[rootFolder].Value, usnHelpers[rootFolder].USN), null);
//Just take it all again
usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_filter, m_unproccesed.Callback);
}
else //All good we rely on USN numbers to find a list of changed files
{
//Find all changed files and folders
Dictionary<string, string> tmp = new Dictionary<string, string>(Utility.Utility.ClientFilenameStringComparer);
foreach (string sx in usnHelpers[rootFolder].GetChangedFileSystemEntries(s, m_lastUSN[rootFolder].Value))
{
tmp.Add(sx, null);
if (!sx.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
m_unproccesed.Files.Add(sx);
}
//Remove the existing unchanged ones
foreach (string x in usnHelpers[rootFolder].GetFileSystemEntries(s))
if (!tmp.ContainsKey(x))
unchanged.Add(x);
}
}
else */
{
usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_unproccesed.Callback);
}
}
}
if (usnHelpers.Count > 0)
{
m_currentUSN = new USNRecord();
foreach (KeyValuePair<string, Snapshots.USNHelper> kx in usnHelpers)
m_currentUSN.Add(kx.Key, kx.Value.JournalID, kx.Value.USN);
}
}
}
catch (Exception ex)
{
if (options.UsnStrategy == Options.OptimizationStrategy.Required)
throw;
else if (options.UsnStrategy == Options.OptimizationStrategy.On)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.UsnFailedError, ex.ToString()), ex);
}
//If we get here, something went really wrong with USN, so we disable it
m_currentUSN = null;
m_unproccesed = new PathCollector(m_snapshot, options.SymlinkPolicy, options.FileAttributeFilter, m_filter, m_stat);
unchanged = null;
}
finally
{
if (usnHelpers != null)
foreach(Snapshots.USNHelper h in usnHelpers.Values)
try { h.Dispose(); }
catch (Exception ex)
{
if (m_stat != null)
m_stat.LogWarning(string.Format(Strings.RSyncDir.UsnDisposeFailedWarning, ex.ToString()), ex);
}
}
if (m_currentUSN == null)
{
m_snapshot.EnumerateFilesAndFolders(m_unproccesed.Callback);
}
else
{
//Skip all items that we know are unchanged
foreach (string x in unchanged)
{
string relpath = GetRelativeName(x);
m_oldSignatures.Remove(relpath);
m_oldFolders.Remove(relpath);
m_oldSymlinks.Remove(relpath);
}
//If some folders did not support USN, add their files now
foreach (string s in m_sourcefolder)
if (!m_currentUSN.ContainsKey(System.IO.Path.GetPathRoot(s)))
m_snapshot.EnumerateFilesAndFolders(s, m_unproccesed.Callback);
}
m_totalfiles = m_unproccesed.Files.Count;
m_isfirstmultipass = true;
if (options.ExcludeEmptyFolders)
{
//We remove the folders that have no files.
//It would be more optimal to exclude them from the list before this point,
// but that would require rewriting of the snapshots
//We can't rely on the order of the folders, so we sort them to get the shortest folder names first
m_unproccesed.Folders.Sort(Utility.Utility.ClientFilenameStringComparer);
//We can't rely on the order of the files either, but sorting them allows us to use a O=log(n) search rather than O=n
m_unproccesed.Files.Sort(Utility.Utility.ClientFilenameStringComparer);
for (int i = 0; i < m_unproccesed.Folders.Count; i++)
{
string folder = m_unproccesed.Folders[i];
int ix = m_unproccesed.Files.BinarySearch(folder, Utility.Utility.ClientFilenameStringComparer);
if (ix >= 0)
continue; //Should not happen, means that a file has the same name as a folder
//Get the next index larger than the foldername
ix = ~ix;
if (ix >= m_unproccesed.Files.Count)
{
//No files matched
m_unproccesed.Folders.RemoveAt(i);
i--;
}
else
{
//If the element does not start with the foldername, no files from the folder are included
if (!m_unproccesed.Files[ix].StartsWith(folder, Utility.Utility.ClientFilenameStringComparision))
{
//Speedup, remove all subfolders as well without performing binary searches
while (i < m_unproccesed.Folders.Count && m_unproccesed.Folders[i].StartsWith(folder))
m_unproccesed.Folders.RemoveAt(i);
//We have removed at least one, so adjust the loop counter
i--;
}
}
}
}
//Build folder diffs
foreach(string s in m_unproccesed.Folders)
{
try
{
string relpath = GetRelativeName(s);
if (relpath.Trim().Length != 0)
{
DateTime lastWrite = m_snapshot.GetLastWriteTime(s).ToUniversalTime();
//Cut off as we only have seconds stored
lastWrite = new DateTime(lastWrite.Year, lastWrite.Month, lastWrite.Day, lastWrite.Hour, lastWrite.Minute, lastWrite.Second, DateTimeKind.Utc);
if (!m_oldFolders.ContainsKey(relpath))
m_newfolders.Add(new KeyValuePair<string, DateTime>(relpath, lastWrite));
else
{
if (m_oldFolders[relpath] != lastWrite)
m_updatedfolders.Add(new KeyValuePair<string, DateTime>(relpath, lastWrite));
m_oldFolders.Remove(relpath);
}
}
}
catch (Exception ex)
{
m_unproccesed.Errors.Add(s);
m_stat.LogError(string.Format(Strings.RSyncDir.FolderModificationTimeReadError, s, ex.Message), ex);
}
}
m_unproccesed.Folders.Clear();
foreach(string s in m_oldFolders.Keys)
if (!m_unproccesed.IsAffectedByError(s))
m_deletedfolders.Add(s);
//Build symlink diffs
if (m_oldSymlinks.Count > 0)
{
for (int i = m_unproccesed.Symlinks.Count - 1; i >= 0; i--)
{
string s = m_unproccesed.Symlinks[i].Key;
try
{
string relpath = GetRelativeName(s);
if (relpath.Trim().Length != 0)
{
string oldLink;
if (m_oldSymlinks.TryGetValue(relpath, out oldLink))
{
m_oldSymlinks.Remove(relpath);
if (string.Equals(oldLink, m_unproccesed.Symlinks[i].Value, Utility.Utility.ClientFilenameStringComparision))
{
m_unproccesed.Symlinks.RemoveAt(i);
}
}
}
}
catch (Exception ex)
{
m_unproccesed.Errors.Add(s);
m_stat.LogError(string.Format(Strings.RSyncDir.SymlinkReadError, s, ex.Message), ex);
}
}
}
m_deletedfiles.AddRange(m_oldSymlinks.Keys);
m_oldSymlinks.Clear();
m_sortedfilelist = options.SortedFilelist;
if (m_sortedfilelist)
m_unproccesed.Files.Sort(Utility.Utility.ClientFilenameStringComparer);
}
public MFTSearchRecord(USNRecord usn)
{
this.usn = usn;
}