void MapManifest(Manifest.Folder Folder, ref List <Manifest.File> Results, ref List <ArchiveFilename> ArchivesRequired, ref long TotalSize, ref Continuation Continuation, ref int MapStartTick)
{
if (Continuation.Required)
{
return;
}
if (Continuation.Starting)
{
if (!Utility.IsContainedIn(Folder.RelativePath, Continuation.LastRelativePath))
{
return;
}
}
foreach (Manifest.Folder Subfolder in Folder.Folders)
{
ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\tMapping manifest folder '" + Subfolder.RelativePath + "'...");
MapManifest(Subfolder, ref Results, ref ArchivesRequired, ref TotalSize, ref Continuation, ref MapStartTick);
DoEvents();
if (Continuation.Required)
{
return;
}
}
long FilesAdded = 0, ArchivesAdded = 0;
foreach (Manifest.File File in Folder.Files)
{
if (Continuation.Starting)
{
if (File.RelativePath.Equals(Continuation.LastRelativePath, System.StringComparison.OrdinalIgnoreCase))
{
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Found continuation marker '" + Continuation.LastRelativePath + ".");
Continuation.Starting = false;
MapStartTick = Environment.TickCount;
}
continue; // The marker file itself was included in the previous verification run, not this one.
}
Results.Add(File);
TotalSize += (long)File.Length;
FilesAdded++;
ArchiveFilename Archive = ArchiveFilename.Parse(File.ArchiveFile);
if (!ArchivesRequired.Contains(Archive))
{
ArchivesRequired.Add(Archive); ArchivesAdded++;
}
if (AutomaticVerify && (Environment.TickCount - MapStartTick) > MaxMappingDurationInTicks)
{
Continuation.Required = true;
Continuation.LastRelativePath = File.RelativePath;
ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\tMapping time limit exceeded. Marking mapping continuation at '" + Continuation.LastRelativePath + "'.");
return;
}
}
ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\t\tFound " + FilesAdded + " files and " + ArchivesAdded + " new archives to be verified.");
}
void IdentifyRequiredArchives(List <ArchiveFilename> ArchivesRequired, Manifest.Entry Entry)
{
try
{
if (Entry is Manifest.File)
{
Manifest.File File = (Manifest.File)Entry;
TotalBytes += (long)File.Length;
ArchiveFilename Archive = ArchiveFilename.Parse(File.ArchiveFile);
if (!ArchivesRequired.Contains(Archive))
{
ArchivesRequired.Add(Archive);
}
}
else if (Entry is Manifest.Folder)
{
Manifest.Folder Folder = (Manifest.Folder)Entry;
foreach (Manifest.File File in Folder.Files)
{
IdentifyRequiredArchives(ArchivesRequired, File);
}
foreach (Manifest.Folder Subfolder in Folder.Folders)
{
IdentifyRequiredArchives(ArchivesRequired, Subfolder);
}
}
else
{
throw new ArgumentException();
}
}
catch (CancelException ce) { throw ce; }
catch (Exception ex)
{
try
{
ZippyForm.LogWriteLine(LogLevel.Information, "Error while identifying required archive for '" + Entry.RelativePath + "': " + ex.Message);
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Detailed error: " + ex.ToString());
throw new Exception(ex.Message + "\nWhile analyzing archives for entry '" + Entry.RelativePath + "'.", ex);
}
catch (Exception exc)
{
throw new Exception(exc.Message + "\nWhile generating error message for: \n\n" + ex.Message, exc);
}
}
}
void Run(ref Continuation Continuation)
{
// Setup progress UI...
Progress = new ProgressForm();
Progress.Text = "Verifying archived files for project '" + Project.Name + "'.";
Progress.OverallProgressBar.Maximum = 10000;
Progress.OverallProgressBar.Minimum = 0;
Progress.OverallProgressBar.Value = 0;
Progress.CancelPrompt = "Are you sure you wish to cancel this verification operation?";
Progress.Show();
try
{
System.Diagnostics.Process.GetCurrentProcess().PriorityClass = System.Diagnostics.ProcessPriorityClass.BelowNormal;
ZippyForm.LogWriteLine(LogLevel.Information, "Starting verification of backup '" + Project.Name + "'.");
string UserTempFolder = Path.GetTempPath();
TempRoot = new DirectoryInfo(Utility.StripTrailingSlash(UserTempFolder) + "\\ZippyBackup_Verification");
Directory.CreateDirectory(TempRoot.FullName);
try
{
Progress.label1.Text = "Retrieving latest archive manifest...";
DoEvents();
ArchiveFilename LatestBackup;
Manifest LatestManifest;
try
{
using (NetworkConnection newself = new NetworkConnection(Project.CompleteBackupFolder, Project.BackupCredentials))
{
// Locate latest backup (either complete or incremental.)
lock (Project.ArchiveFileList) LatestBackup = Project.ArchiveFileList.FindMostRecent();
if (LatestBackup == ArchiveFilename.MaxValue)
{
throw new NoCompleteBackupException();
}
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Loading most recent manifest from '" + LatestBackup.ToString() + "' of project '" + Project.Name + "'.");
LatestManifest = LatestBackup.LoadArchiveManifest(Project, true);
}
}
catch (CancelException ce) { throw ce; }
catch (Exception ex)
{
throw new Exception(ex.Message + " Error while loading last backup for incremental update.", ex);
}
int StartTick = Environment.TickCount;
if (Continuation.Starting)
{
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Continuing from previous run. Searching for starting point '" + Continuation.LastRelativePath + "'.");
}
Continuation MapContinuation = new Continuation();
MapContinuation.Starting = Continuation.Starting;
MapContinuation.LastRelativePath = Continuation.LastRelativePath;
Continuation.Required = false;
while (!Continuation.Required)
{
/** Operate on a single zip file at a time. This requires planning which files are
* coming from where. Also, we count how many total bytes we'll be extracting. **/
Progress.label1.Text = "Identifying required archive files...";
DoEvents();
List <Manifest.File> FileList = new List <Manifest.File>();
List <ArchiveFilename> ArchivesRequired = new List <ArchiveFilename>();
TotalBytes = 0;
/** For extremely large archives, it can happen that even just scanning through the manifest can be very time consuming. We don't want
* to spend too much time on this step, so we have a time limit for coming up with file/archive list. But we also have a time limit
* for the overall verify operation. We use a pattern of spending N minutes coming up with filenames and then processing that bunch,
* then repeating as long as the overall M minute limit hasn't yet been reached. This requires keeping track of "where we left off"
* for both the scanning operation and the overall verify operation. If a file hasn't yet been extracted and verified and we have to
* quit because of time or cancellation, then we need the overall "where we left off" to point to the last extracted file, potentially
* repeating a little bit of the mapping time when we startup again.
*/
int MapStartTick = Environment.TickCount;
// If the last loop ran into a mapping time limit, initiate a continuation on the mapping. If the last verification ran into a time limit,
// we are also doing a mapping continuation because there is no need to map anything before the first file of the new set.
if (MapContinuation.Required)
{
MapContinuation.Starting = true;
}
MapContinuation.Required = false;
if (MapContinuation.Starting)
{
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Identifying required archive files, beginning at '" + MapContinuation.LastRelativePath + "'...");
}
else
{
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Identifying required archive files...");
}
MapManifest(LatestManifest.ArchiveRoot, ref FileList, ref ArchivesRequired, ref TotalBytes, ref MapContinuation, ref MapStartTick);
if (MapContinuation.Starting)
{
// We have the case where we never found the continuation marker. This can happen if the latest archive mismatches
// the one where the continuation marker was made. We just restart from the beginning of the archive.
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Continuation marker not found. Starting verification from beginning.");
MapContinuation.Starting = false;
Debug.Assert(FileList.Count == 0 && ArchivesRequired.Count == 0);
MapStartTick = Environment.TickCount;
MapManifest(LatestManifest.ArchiveRoot, ref FileList, ref ArchivesRequired, ref TotalBytes, ref MapContinuation, ref MapStartTick);
}
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Archives requiring verification (" + ArchivesRequired.Count + "):");
foreach (ArchiveFilename Archive in ArchivesRequired)
{
ZippyForm.LogWriteLine(LogLevel.LightDebug, "\t" + Archive.ToString());
}
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Starting individual verifications of " + FileList.Count + " files...");
// Begin extracting each file and then immediately deleting if successful.
int FilesVerified = 0;
foreach (ArchiveFilename Archive in ArchivesRequired)
{
Progress.label1.Text = "Extracting from archive: " + Archive.ToString();
DoEvents();
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verifying archive: " + Archive.ToString());
try
{
using (ZipFile zip = ZipFile.Read(Project.CompleteBackupFolder + "\\" + Archive.ToString()))
{
zip.ZipError += new EventHandler <ZipErrorEventArgs>(OnZipError);
zip.ExtractProgress += new EventHandler <ExtractProgressEventArgs>(OnZipExtractProgress);
foreach (Manifest.File Entry in FileList)
{
ArchiveFilename NeededArchive = ArchiveFilename.Parse(Entry.ArchiveFile);
if (NeededArchive != Archive)
{
continue;
}
RunEntry(zip, Archive, Entry, Utility.StripTrailingSlash(TempRoot.FullName));
FilesVerified++;
if (AutomaticVerify && (Environment.TickCount - StartTick) > MaxAutomaticDurationInTicks)
{
Continuation.Required = true;
Continuation.Starting = true;
Continuation.LastRelativePath = Entry.RelativePath;
ZippyForm.LogWriteLine(LogLevel.LightDebug, "New verification continuation marker set to '" + Continuation.LastRelativePath + "'.");
ZippyForm.LogWriteLine(LogLevel.Information, "Verify terminating early due to individual scan time limit.");
break;
}
}
}
}
catch (FileNotFoundException fe)
{
StringBuilder Msg = new StringBuilder();
Msg.Append("Unable to locate file(s) in archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + fe.Message);
switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel))
{
case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break;
case DialogResult.No: break;
case DialogResult.Cancel: throw new CancelException();
default: throw new NotSupportedException();
}
}
catch (Ionic.Zip.ZipException ze)
{
StringBuilder Msg = new StringBuilder();
Msg.Append("Unable to extract file(s) from archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + ze.Message);
switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel))
{
case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break;
case DialogResult.No: break;
case DialogResult.Cancel: throw new CancelException();
default: throw new NotSupportedException();
}
}
catch (Ionic.Zlib.ZlibException ze)
{
StringBuilder Msg = new StringBuilder();
Msg.Append("Unable to extract file(s) from archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + ze.Message);
switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel))
{
case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break;
case DialogResult.No: break;
case DialogResult.Cancel: throw new CancelException();
default: throw new NotSupportedException();
}
}
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verified " + FilesVerified + " of " + FileList.Count + " files.");
if (Continuation.Required)
{
break;
}
}
if (!MapContinuation.Required)
{
break;
}
}
string BackupStatusFile = Project.CompleteBackupFolder + "\\Backup_Status.xml";
BackupStatus bs = new BackupStatus();
try { bs = BackupStatus.Load(BackupStatusFile); }
catch (Exception) { }
bs.LastVerify = DateTime.UtcNow;
if (!Continuation.Required)
{
bs.LastCompletedVerify = DateTime.UtcNow;
}
if (!Continuation.Required)
{
bs.LastVerifyRelativePath = "";
}
else
{
bs.LastVerifyRelativePath = Continuation.LastRelativePath;
}
bs.Save(BackupStatusFile);
ZippyForm.LogWriteLine(LogLevel.Information, "Verification complete.");
}
finally
{
Directory.Delete(TempRoot.FullName);
}
}
catch (CancelException ce)
{
ZippyForm.LogWriteLine(LogLevel.Information, "Verification cancelled by user before completion.");
throw ce;
}
catch (Exception ex)
{
ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verification interrupted by error: " + ex.Message);
throw ex;
}
finally
{
Progress.Dispose();
Progress = null;
System.Diagnostics.Process.GetCurrentProcess().PriorityClass = System.Diagnostics.ProcessPriorityClass.Normal;
}
}
void RunEntry(ZipFile zip, ArchiveFilename Archive, Manifest.Entry Entry, string DestinationFolder)
{
if (Entry == null || String.IsNullOrEmpty(DestinationFolder))
{
throw new ArgumentException("Invalid manifest entry or processing error for manifest entry.", "Entry");
}
bool IsArchiveRoot = (String.IsNullOrEmpty(Entry.Name)) && Entry is Manifest.Folder;
Progress.label2.Text = "Extracting: " + Entry.RelativePath;
DoEvents();
string Path = Utility.StripTrailingSlash(DestinationFolder) + "\\" + Entry.Name;
if (Entry is Manifest.File)
{
Manifest.File File = (Manifest.File)Entry;
try
{
ArchiveFilename NeededArchive = ArchiveFilename.Parse(File.ArchiveFile);
if (NeededArchive == Archive)
{
Directory.CreateDirectory(DestinationFolder);
ExtractFile(zip, File, Path);
ExtractProperties(File, Path);
BytesCompleted += (long)File.Length;
}
}
catch (CancelException ce) { throw ce; }
catch (Exception ex)
{
throw new Exception(ex.Message + "\nWhile extracting file '" + Entry.RelativePath + "'.", ex);
}
}
else if (Entry is Manifest.Folder)
{
Manifest.Folder Folder = (Manifest.Folder)Entry;
try
{
Directory.CreateDirectory(Path);
foreach (Manifest.File File in Folder.Files)
{
RunEntry(zip, Archive, File, Path);
}
foreach (Manifest.Folder Subfolder in Folder.Folders)
{
RunEntry(zip, Archive, Subfolder, Path);
}
if (!IsArchiveRoot)
{
ExtractProperties(Folder, Path);
}
}
catch (CancelException ce) { throw ce; }
catch (Exception ex)
{
throw new Exception(ex.Message + "\nWhile extracting folder '" + Entry.RelativePath + "'.", ex);
}
}
else
{
throw new ArgumentException();
}
Progress.OverallProgressBar.Value = (int)(10000L * BytesCompleted / TotalBytes);
return;
}
