void MapManifest(Manifest.Folder Folder, ref List <Manifest.File> Results, ref List <ArchiveFilename> ArchivesRequired, ref long TotalSize, ref Continuation Continuation, ref int MapStartTick) { if (Continuation.Required) { return; } if (Continuation.Starting) { if (!Utility.IsContainedIn(Folder.RelativePath, Continuation.LastRelativePath)) { return; } } foreach (Manifest.Folder Subfolder in Folder.Folders) { ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\tMapping manifest folder '" + Subfolder.RelativePath + "'..."); MapManifest(Subfolder, ref Results, ref ArchivesRequired, ref TotalSize, ref Continuation, ref MapStartTick); DoEvents(); if (Continuation.Required) { return; } } long FilesAdded = 0, ArchivesAdded = 0; foreach (Manifest.File File in Folder.Files) { if (Continuation.Starting) { if (File.RelativePath.Equals(Continuation.LastRelativePath, System.StringComparison.OrdinalIgnoreCase)) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Found continuation marker '" + Continuation.LastRelativePath + "."); Continuation.Starting = false; MapStartTick = Environment.TickCount; } continue; // The marker file itself was included in the previous verification run, not this one. } Results.Add(File); TotalSize += (long)File.Length; FilesAdded++; ArchiveFilename Archive = ArchiveFilename.Parse(File.ArchiveFile); if (!ArchivesRequired.Contains(Archive)) { ArchivesRequired.Add(Archive); ArchivesAdded++; } if (AutomaticVerify && (Environment.TickCount - MapStartTick) > MaxMappingDurationInTicks) { Continuation.Required = true; Continuation.LastRelativePath = File.RelativePath; ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\tMapping time limit exceeded. Marking mapping continuation at '" + Continuation.LastRelativePath + "'."); return; } } ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\t\tFound " + FilesAdded + " files and " + ArchivesAdded + " new archives to be verified."); }
void IdentifyRequiredArchives(List <ArchiveFilename> ArchivesRequired, Manifest.Entry Entry) { try { if (Entry is Manifest.File) { Manifest.File File = (Manifest.File)Entry; TotalBytes += (long)File.Length; ArchiveFilename Archive = ArchiveFilename.Parse(File.ArchiveFile); if (!ArchivesRequired.Contains(Archive)) { ArchivesRequired.Add(Archive); } } else if (Entry is Manifest.Folder) { Manifest.Folder Folder = (Manifest.Folder)Entry; foreach (Manifest.File File in Folder.Files) { IdentifyRequiredArchives(ArchivesRequired, File); } foreach (Manifest.Folder Subfolder in Folder.Folders) { IdentifyRequiredArchives(ArchivesRequired, Subfolder); } } else { throw new ArgumentException(); } } catch (CancelException ce) { throw ce; } catch (Exception ex) { try { ZippyForm.LogWriteLine(LogLevel.Information, "Error while identifying required archive for '" + Entry.RelativePath + "': " + ex.Message); ZippyForm.LogWriteLine(LogLevel.LightDebug, "Detailed error: " + ex.ToString()); throw new Exception(ex.Message + "\nWhile analyzing archives for entry '" + Entry.RelativePath + "'.", ex); } catch (Exception exc) { throw new Exception(exc.Message + "\nWhile generating error message for: \n\n" + ex.Message, exc); } } }
void Run(ref Continuation Continuation) { // Setup progress UI... Progress = new ProgressForm(); Progress.Text = "Verifying archived files for project '" + Project.Name + "'."; Progress.OverallProgressBar.Maximum = 10000; Progress.OverallProgressBar.Minimum = 0; Progress.OverallProgressBar.Value = 0; Progress.CancelPrompt = "Are you sure you wish to cancel this verification operation?"; Progress.Show(); try { System.Diagnostics.Process.GetCurrentProcess().PriorityClass = System.Diagnostics.ProcessPriorityClass.BelowNormal; ZippyForm.LogWriteLine(LogLevel.Information, "Starting verification of backup '" + Project.Name + "'."); string UserTempFolder = Path.GetTempPath(); TempRoot = new DirectoryInfo(Utility.StripTrailingSlash(UserTempFolder) + "\\ZippyBackup_Verification"); Directory.CreateDirectory(TempRoot.FullName); try { Progress.label1.Text = "Retrieving latest archive manifest..."; DoEvents(); ArchiveFilename LatestBackup; Manifest LatestManifest; try { using (NetworkConnection newself = new NetworkConnection(Project.CompleteBackupFolder, Project.BackupCredentials)) { // Locate latest backup (either complete or incremental.) lock (Project.ArchiveFileList) LatestBackup = Project.ArchiveFileList.FindMostRecent(); if (LatestBackup == ArchiveFilename.MaxValue) { throw new NoCompleteBackupException(); } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Loading most recent manifest from '" + LatestBackup.ToString() + "' of project '" + Project.Name + "'."); LatestManifest = LatestBackup.LoadArchiveManifest(Project, true); } } catch (CancelException ce) { throw ce; } catch (Exception ex) { throw new Exception(ex.Message + " Error while loading last backup for incremental update.", ex); } int StartTick = Environment.TickCount; if (Continuation.Starting) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Continuing from previous run. Searching for starting point '" + Continuation.LastRelativePath + "'."); } Continuation MapContinuation = new Continuation(); MapContinuation.Starting = Continuation.Starting; MapContinuation.LastRelativePath = Continuation.LastRelativePath; Continuation.Required = false; while (!Continuation.Required) { /** Operate on a single zip file at a time. This requires planning which files are * coming from where. Also, we count how many total bytes we'll be extracting. **/ Progress.label1.Text = "Identifying required archive files..."; DoEvents(); List <Manifest.File> FileList = new List <Manifest.File>(); List <ArchiveFilename> ArchivesRequired = new List <ArchiveFilename>(); TotalBytes = 0; /** For extremely large archives, it can happen that even just scanning through the manifest can be very time consuming. We don't want * to spend too much time on this step, so we have a time limit for coming up with file/archive list. But we also have a time limit * for the overall verify operation. We use a pattern of spending N minutes coming up with filenames and then processing that bunch, * then repeating as long as the overall M minute limit hasn't yet been reached. This requires keeping track of "where we left off" * for both the scanning operation and the overall verify operation. If a file hasn't yet been extracted and verified and we have to * quit because of time or cancellation, then we need the overall "where we left off" to point to the last extracted file, potentially * repeating a little bit of the mapping time when we startup again. */ int MapStartTick = Environment.TickCount; // If the last loop ran into a mapping time limit, initiate a continuation on the mapping. If the last verification ran into a time limit, // we are also doing a mapping continuation because there is no need to map anything before the first file of the new set. if (MapContinuation.Required) { MapContinuation.Starting = true; } MapContinuation.Required = false; if (MapContinuation.Starting) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Identifying required archive files, beginning at '" + MapContinuation.LastRelativePath + "'..."); } else { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Identifying required archive files..."); } MapManifest(LatestManifest.ArchiveRoot, ref FileList, ref ArchivesRequired, ref TotalBytes, ref MapContinuation, ref MapStartTick); if (MapContinuation.Starting) { // We have the case where we never found the continuation marker. This can happen if the latest archive mismatches // the one where the continuation marker was made. We just restart from the beginning of the archive. ZippyForm.LogWriteLine(LogLevel.LightDebug, "Continuation marker not found. Starting verification from beginning."); MapContinuation.Starting = false; Debug.Assert(FileList.Count == 0 && ArchivesRequired.Count == 0); MapStartTick = Environment.TickCount; MapManifest(LatestManifest.ArchiveRoot, ref FileList, ref ArchivesRequired, ref TotalBytes, ref MapContinuation, ref MapStartTick); } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Archives requiring verification (" + ArchivesRequired.Count + "):"); foreach (ArchiveFilename Archive in ArchivesRequired) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "\t" + Archive.ToString()); } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Starting individual verifications of " + FileList.Count + " files..."); // Begin extracting each file and then immediately deleting if successful. int FilesVerified = 0; foreach (ArchiveFilename Archive in ArchivesRequired) { Progress.label1.Text = "Extracting from archive: " + Archive.ToString(); DoEvents(); ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verifying archive: " + Archive.ToString()); try { using (ZipFile zip = ZipFile.Read(Project.CompleteBackupFolder + "\\" + Archive.ToString())) { zip.ZipError += new EventHandler <ZipErrorEventArgs>(OnZipError); zip.ExtractProgress += new EventHandler <ExtractProgressEventArgs>(OnZipExtractProgress); foreach (Manifest.File Entry in FileList) { ArchiveFilename NeededArchive = ArchiveFilename.Parse(Entry.ArchiveFile); if (NeededArchive != Archive) { continue; } RunEntry(zip, Archive, Entry, Utility.StripTrailingSlash(TempRoot.FullName)); FilesVerified++; if (AutomaticVerify && (Environment.TickCount - StartTick) > MaxAutomaticDurationInTicks) { Continuation.Required = true; Continuation.Starting = true; Continuation.LastRelativePath = Entry.RelativePath; ZippyForm.LogWriteLine(LogLevel.LightDebug, "New verification continuation marker set to '" + Continuation.LastRelativePath + "'."); ZippyForm.LogWriteLine(LogLevel.Information, "Verify terminating early due to individual scan time limit."); break; } } } } catch (FileNotFoundException fe) { StringBuilder Msg = new StringBuilder(); Msg.Append("Unable to locate file(s) in archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + fe.Message); switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel)) { case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break; case DialogResult.No: break; case DialogResult.Cancel: throw new CancelException(); default: throw new NotSupportedException(); } } catch (Ionic.Zip.ZipException ze) { StringBuilder Msg = new StringBuilder(); Msg.Append("Unable to extract file(s) from archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + ze.Message); switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel)) { case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break; case DialogResult.No: break; case DialogResult.Cancel: throw new CancelException(); default: throw new NotSupportedException(); } } catch (Ionic.Zlib.ZlibException ze) { StringBuilder Msg = new StringBuilder(); Msg.Append("Unable to extract file(s) from archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + ze.Message); switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel)) { case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break; case DialogResult.No: break; case DialogResult.Cancel: throw new CancelException(); default: throw new NotSupportedException(); } } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verified " + FilesVerified + " of " + FileList.Count + " files."); if (Continuation.Required) { break; } } if (!MapContinuation.Required) { break; } } string BackupStatusFile = Project.CompleteBackupFolder + "\\Backup_Status.xml"; BackupStatus bs = new BackupStatus(); try { bs = BackupStatus.Load(BackupStatusFile); } catch (Exception) { } bs.LastVerify = DateTime.UtcNow; if (!Continuation.Required) { bs.LastCompletedVerify = DateTime.UtcNow; } if (!Continuation.Required) { bs.LastVerifyRelativePath = ""; } else { bs.LastVerifyRelativePath = Continuation.LastRelativePath; } bs.Save(BackupStatusFile); ZippyForm.LogWriteLine(LogLevel.Information, "Verification complete."); } finally { Directory.Delete(TempRoot.FullName); } } catch (CancelException ce) { ZippyForm.LogWriteLine(LogLevel.Information, "Verification cancelled by user before completion."); throw ce; } catch (Exception ex) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verification interrupted by error: " + ex.Message); throw ex; } finally { Progress.Dispose(); Progress = null; System.Diagnostics.Process.GetCurrentProcess().PriorityClass = System.Diagnostics.ProcessPriorityClass.Normal; } }
void RunEntry(ZipFile zip, ArchiveFilename Archive, Manifest.Entry Entry, string DestinationFolder) { if (Entry == null || String.IsNullOrEmpty(DestinationFolder)) { throw new ArgumentException("Invalid manifest entry or processing error for manifest entry.", "Entry"); } bool IsArchiveRoot = (String.IsNullOrEmpty(Entry.Name)) && Entry is Manifest.Folder; Progress.label2.Text = "Extracting: " + Entry.RelativePath; DoEvents(); string Path = Utility.StripTrailingSlash(DestinationFolder) + "\\" + Entry.Name; if (Entry is Manifest.File) { Manifest.File File = (Manifest.File)Entry; try { ArchiveFilename NeededArchive = ArchiveFilename.Parse(File.ArchiveFile); if (NeededArchive == Archive) { Directory.CreateDirectory(DestinationFolder); ExtractFile(zip, File, Path); ExtractProperties(File, Path); BytesCompleted += (long)File.Length; } } catch (CancelException ce) { throw ce; } catch (Exception ex) { throw new Exception(ex.Message + "\nWhile extracting file '" + Entry.RelativePath + "'.", ex); } } else if (Entry is Manifest.Folder) { Manifest.Folder Folder = (Manifest.Folder)Entry; try { Directory.CreateDirectory(Path); foreach (Manifest.File File in Folder.Files) { RunEntry(zip, Archive, File, Path); } foreach (Manifest.Folder Subfolder in Folder.Folders) { RunEntry(zip, Archive, Subfolder, Path); } if (!IsArchiveRoot) { ExtractProperties(Folder, Path); } } catch (CancelException ce) { throw ce; } catch (Exception ex) { throw new Exception(ex.Message + "\nWhile extracting folder '" + Entry.RelativePath + "'.", ex); } } else { throw new ArgumentException(); } Progress.OverallProgressBar.Value = (int)(10000L * BytesCompleted / TotalBytes); return; }