void RunEntry(ZipFile zip, ArchiveFilename Archive, Manifest.File File, string TempFolder) { if (File == null || String.IsNullOrEmpty(TempFolder)) { throw new ArgumentException("Invalid manifest entry or processing error for manifest entry.", "Entry"); } Progress.label2.Text = "Extracting: " + File.RelativePath; DoEvents(); string TempPath = Utility.StripTrailingSlash(TempFolder) + "\\" + File.Name; try { ZippyForm.LogWriteLine(LogLevel.MediumDebug, "Verifying (extracting) file '" + File.RelativePath + "' to temporary path '" + TempPath + "'."); ExtractAndDeleteFile(zip, File, TempPath); BytesCompleted += (long)File.Length; } catch (CancelException ce) { throw ce; } catch (Ionic.Zip.ZipException ze) { throw ze; } catch (Ionic.Zlib.ZlibException ze) { throw ze; } catch (FileNotFoundException fe) { throw fe; } catch (Exception ex) { throw new Exception(ex.Message + "\nWhile verifying file '" + File.RelativePath + "' by trial extraction.", ex); } Progress.OverallProgressBar.Value = (int)(10000L * BytesCompleted / TotalBytes); return; }
void MapManifest(Manifest.Folder Folder, ref List <Manifest.File> Results, ref List <ArchiveFilename> ArchivesRequired, ref long TotalSize, ref Continuation Continuation, ref int MapStartTick) { if (Continuation.Required) { return; } if (Continuation.Starting) { if (!Utility.IsContainedIn(Folder.RelativePath, Continuation.LastRelativePath)) { return; } } foreach (Manifest.Folder Subfolder in Folder.Folders) { ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\tMapping manifest folder '" + Subfolder.RelativePath + "'..."); MapManifest(Subfolder, ref Results, ref ArchivesRequired, ref TotalSize, ref Continuation, ref MapStartTick); DoEvents(); if (Continuation.Required) { return; } } long FilesAdded = 0, ArchivesAdded = 0; foreach (Manifest.File File in Folder.Files) { if (Continuation.Starting) { if (File.RelativePath.Equals(Continuation.LastRelativePath, System.StringComparison.OrdinalIgnoreCase)) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Found continuation marker '" + Continuation.LastRelativePath + "."); Continuation.Starting = false; MapStartTick = Environment.TickCount; } continue; // The marker file itself was included in the previous verification run, not this one. } Results.Add(File); TotalSize += (long)File.Length; FilesAdded++; ArchiveFilename Archive = ArchiveFilename.Parse(File.ArchiveFile); if (!ArchivesRequired.Contains(Archive)) { ArchivesRequired.Add(Archive); ArchivesAdded++; } if (AutomaticVerify && (Environment.TickCount - MapStartTick) > MaxMappingDurationInTicks) { Continuation.Required = true; Continuation.LastRelativePath = File.RelativePath; ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\tMapping time limit exceeded. Marking mapping continuation at '" + Continuation.LastRelativePath + "'."); return; } } ZippyForm.LogWriteLine(LogLevel.MediumDebug, "\t\tFound " + FilesAdded + " files and " + ArchivesAdded + " new archives to be verified."); }
public override bool Equals(object obj) { ArchiveFilename b = obj as ArchiveFilename; if ((object)b == null) { return(false); } return(BackupDate == b.BackupDate && BackupTime == b.BackupTime && BackupType == b.BackupType); }
void IdentifyRequiredArchives(List <ArchiveFilename> ArchivesRequired, Manifest.Entry Entry) { try { if (Entry is Manifest.File) { Manifest.File File = (Manifest.File)Entry; TotalBytes += (long)File.Length; ArchiveFilename Archive = ArchiveFilename.Parse(File.ArchiveFile); if (!ArchivesRequired.Contains(Archive)) { ArchivesRequired.Add(Archive); } } else if (Entry is Manifest.Folder) { Manifest.Folder Folder = (Manifest.Folder)Entry; foreach (Manifest.File File in Folder.Files) { IdentifyRequiredArchives(ArchivesRequired, File); } foreach (Manifest.Folder Subfolder in Folder.Folders) { IdentifyRequiredArchives(ArchivesRequired, Subfolder); } } else { throw new ArgumentException(); } } catch (CancelException ce) { throw ce; } catch (Exception ex) { try { ZippyForm.LogWriteLine(LogLevel.Information, "Error while identifying required archive for '" + Entry.RelativePath + "': " + ex.Message); ZippyForm.LogWriteLine(LogLevel.LightDebug, "Detailed error: " + ex.ToString()); throw new Exception(ex.Message + "\nWhile analyzing archives for entry '" + Entry.RelativePath + "'.", ex); } catch (Exception exc) { throw new Exception(exc.Message + "\nWhile generating error message for: \n\n" + ex.Message, exc); } } }
/// <summary> /// WasUpdatedInArchive() examines whether any files or subfolders of a specified folder /// within an archive actually contained any files within this archive. Since a manifest /// contains a complete listing of all files and folders, including ones which are backed /// up by "external reference" (incremental backup), we must figure out if the file is /// actually present inside this archive when searching for it. /// </summary> /// <param name="Archive"></param> /// <param name="ManifestFolder"></param> /// <returns></returns> bool WasUpdatedInArchive(ArchiveFilename Archive, Manifest.Folder ManifestFolder) { foreach (Manifest.File File in ManifestFolder.Files) { if (File.ArchiveFile == Archive.ToString()) { return(true); } } foreach (Manifest.Folder Subfolder in ManifestFolder.Folders) { if (WasUpdatedInArchive(Archive, Subfolder)) { return(true); } } return(false); }
public void LoadAll(BackupProject Project) { Archives = new List <ArchiveFilename>(); try { FileInfo[] FileList = new DirectoryInfo(Project.CompleteBackupFolder).GetFiles("Backup_*.zip"); foreach (FileInfo fi in FileList) { ArchiveFilename af; if (!ArchiveFilename.TryParse(fi.Name, out af)) { continue; } Archives.Add(af); } } catch (DirectoryNotFoundException) { return; } }
bool SearchManifestForFolderName(ArchiveFilename Archive, Manifest.Folder ManifestFolder, string[] SearchWords) { foreach (Manifest.Folder Subfolder in ManifestFolder.Folders) { bool Match = true; foreach (string Word in SearchWords) { if (!Subfolder.Name.ToLowerInvariant().Contains(Word)) { Match = false; break; } } if (Match && WasUpdatedInArchive(Archive, Subfolder)) { return(true); } if (SearchManifestForFolderName(Archive, Subfolder, SearchWords)) { return(true); } } return(false); }
/// <summary> /// TryParse() converts a filename in the ZippyBackup format to a date and time indicator. /// The time indicator is a sequence counter (i.e. A, B, C, etc.) as opposed to an exact time. The /// exact time is available from the XML Manifest inside the zip file. /// </summary> /// <param name="Filename">ZippyBackup format filename to parse.</param> /// <returns>True if the filename was parsed successfully. False otherwise. An exception is thrown /// if a file is identified as a backup zip file but we are otherwise unable to parse the filename.</returns> public static bool TryParse(string Filename, out ArchiveFilename Result) { Result = new ArchiveFilename(); if (!Filename.ToLowerInvariant().StartsWith("backup_")) { return(false); } if (!Filename.ToLowerInvariant().EndsWith(".zip")) { return(false); } // Backup_Jan21A_1980.zip // Backup_Mar18B_1981_Complete.zip // Backup_Jan21BB_1980.zip // 012345678901234567890123456 if (Filename.Length < 22) { return(false); } // Locate the index of the first underscore. int iUnderscore = Filename.IndexOf('_'); if (iUnderscore < 0 || iUnderscore + 1 >= Filename.Length) { throw new FormatException("Unable to locate second underscore in filename within project backup directory: " + Filename); } // Locate the index of the second underscore, place it into iUnderscore for later use. int ii = Filename.Substring(iUnderscore + 1).IndexOf('_'); iUnderscore += ii + 1; if (ii < 0 || iUnderscore + 4 >= Filename.Length) { throw new FormatException("Invalid backup filename format in project backup directory: " + Filename); } // Parse the month abbreviation as a string string MonthAbbr = new string(new char[] { Filename[7], Filename[8], Filename[9] }); int Day, Year; // Parse the day if (!int.TryParse(new string(new char[] { Filename[10], Filename[11] }), out Day)) { throw new FormatException("Invalid day specification in backup filename in project backup directory: " + Filename); } // Parse the time, which is just the sequence letter (i.e. A, B, C, ... AA, AB, AC, etc.) try { Result.BackupTime = Utility.LettersToInt(Filename.Substring(12)); } catch (Exception) { throw new FormatException("Unable to parse backup time (letter) from filename in backup directory file: " + Filename); } // Parse the year if (!int.TryParse(new string(new char[] { Filename[iUnderscore + 1], Filename[iUnderscore + 2], Filename[iUnderscore + 3], Filename[iUnderscore + 4] }), out Year)) { throw new FormatException("Unable to parse year from filename in project backup directory filename: " + Filename); } // Parse the month from abbreviation string int Month = DateTime.ParseExact(MonthAbbr, "MMM", CultureInfo.CurrentCulture).Month; // Combine the parsed date information into a DateTime. Result.BackupDate = new DateTime(Year, Month, Day); // Look for "_Complete" indicator on filename. if (Filename.Contains("_Complete")) { Result.BackupType = BackupTypes.Complete; } else { Result.BackupType = BackupTypes.Incremental; } return(true); }
void Run(ref Continuation Continuation) { // Setup progress UI... Progress = new ProgressForm(); Progress.Text = "Verifying archived files for project '" + Project.Name + "'."; Progress.OverallProgressBar.Maximum = 10000; Progress.OverallProgressBar.Minimum = 0; Progress.OverallProgressBar.Value = 0; Progress.CancelPrompt = "Are you sure you wish to cancel this verification operation?"; Progress.Show(); try { System.Diagnostics.Process.GetCurrentProcess().PriorityClass = System.Diagnostics.ProcessPriorityClass.BelowNormal; ZippyForm.LogWriteLine(LogLevel.Information, "Starting verification of backup '" + Project.Name + "'."); string UserTempFolder = Path.GetTempPath(); TempRoot = new DirectoryInfo(Utility.StripTrailingSlash(UserTempFolder) + "\\ZippyBackup_Verification"); Directory.CreateDirectory(TempRoot.FullName); try { Progress.label1.Text = "Retrieving latest archive manifest..."; DoEvents(); ArchiveFilename LatestBackup; Manifest LatestManifest; try { using (NetworkConnection newself = new NetworkConnection(Project.CompleteBackupFolder, Project.BackupCredentials)) { // Locate latest backup (either complete or incremental.) lock (Project.ArchiveFileList) LatestBackup = Project.ArchiveFileList.FindMostRecent(); if (LatestBackup == ArchiveFilename.MaxValue) { throw new NoCompleteBackupException(); } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Loading most recent manifest from '" + LatestBackup.ToString() + "' of project '" + Project.Name + "'."); LatestManifest = LatestBackup.LoadArchiveManifest(Project, true); } } catch (CancelException ce) { throw ce; } catch (Exception ex) { throw new Exception(ex.Message + " Error while loading last backup for incremental update.", ex); } int StartTick = Environment.TickCount; if (Continuation.Starting) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Continuing from previous run. Searching for starting point '" + Continuation.LastRelativePath + "'."); } Continuation MapContinuation = new Continuation(); MapContinuation.Starting = Continuation.Starting; MapContinuation.LastRelativePath = Continuation.LastRelativePath; Continuation.Required = false; while (!Continuation.Required) { /** Operate on a single zip file at a time. This requires planning which files are * coming from where. Also, we count how many total bytes we'll be extracting. **/ Progress.label1.Text = "Identifying required archive files..."; DoEvents(); List <Manifest.File> FileList = new List <Manifest.File>(); List <ArchiveFilename> ArchivesRequired = new List <ArchiveFilename>(); TotalBytes = 0; /** For extremely large archives, it can happen that even just scanning through the manifest can be very time consuming. We don't want * to spend too much time on this step, so we have a time limit for coming up with file/archive list. But we also have a time limit * for the overall verify operation. We use a pattern of spending N minutes coming up with filenames and then processing that bunch, * then repeating as long as the overall M minute limit hasn't yet been reached. This requires keeping track of "where we left off" * for both the scanning operation and the overall verify operation. If a file hasn't yet been extracted and verified and we have to * quit because of time or cancellation, then we need the overall "where we left off" to point to the last extracted file, potentially * repeating a little bit of the mapping time when we startup again. */ int MapStartTick = Environment.TickCount; // If the last loop ran into a mapping time limit, initiate a continuation on the mapping. If the last verification ran into a time limit, // we are also doing a mapping continuation because there is no need to map anything before the first file of the new set. if (MapContinuation.Required) { MapContinuation.Starting = true; } MapContinuation.Required = false; if (MapContinuation.Starting) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Identifying required archive files, beginning at '" + MapContinuation.LastRelativePath + "'..."); } else { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Identifying required archive files..."); } MapManifest(LatestManifest.ArchiveRoot, ref FileList, ref ArchivesRequired, ref TotalBytes, ref MapContinuation, ref MapStartTick); if (MapContinuation.Starting) { // We have the case where we never found the continuation marker. This can happen if the latest archive mismatches // the one where the continuation marker was made. We just restart from the beginning of the archive. ZippyForm.LogWriteLine(LogLevel.LightDebug, "Continuation marker not found. Starting verification from beginning."); MapContinuation.Starting = false; Debug.Assert(FileList.Count == 0 && ArchivesRequired.Count == 0); MapStartTick = Environment.TickCount; MapManifest(LatestManifest.ArchiveRoot, ref FileList, ref ArchivesRequired, ref TotalBytes, ref MapContinuation, ref MapStartTick); } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Archives requiring verification (" + ArchivesRequired.Count + "):"); foreach (ArchiveFilename Archive in ArchivesRequired) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "\t" + Archive.ToString()); } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Starting individual verifications of " + FileList.Count + " files..."); // Begin extracting each file and then immediately deleting if successful. int FilesVerified = 0; foreach (ArchiveFilename Archive in ArchivesRequired) { Progress.label1.Text = "Extracting from archive: " + Archive.ToString(); DoEvents(); ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verifying archive: " + Archive.ToString()); try { using (ZipFile zip = ZipFile.Read(Project.CompleteBackupFolder + "\\" + Archive.ToString())) { zip.ZipError += new EventHandler <ZipErrorEventArgs>(OnZipError); zip.ExtractProgress += new EventHandler <ExtractProgressEventArgs>(OnZipExtractProgress); foreach (Manifest.File Entry in FileList) { ArchiveFilename NeededArchive = ArchiveFilename.Parse(Entry.ArchiveFile); if (NeededArchive != Archive) { continue; } RunEntry(zip, Archive, Entry, Utility.StripTrailingSlash(TempRoot.FullName)); FilesVerified++; if (AutomaticVerify && (Environment.TickCount - StartTick) > MaxAutomaticDurationInTicks) { Continuation.Required = true; Continuation.Starting = true; Continuation.LastRelativePath = Entry.RelativePath; ZippyForm.LogWriteLine(LogLevel.LightDebug, "New verification continuation marker set to '" + Continuation.LastRelativePath + "'."); ZippyForm.LogWriteLine(LogLevel.Information, "Verify terminating early due to individual scan time limit."); break; } } } } catch (FileNotFoundException fe) { StringBuilder Msg = new StringBuilder(); Msg.Append("Unable to locate file(s) in archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + fe.Message); switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel)) { case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break; case DialogResult.No: break; case DialogResult.Cancel: throw new CancelException(); default: throw new NotSupportedException(); } } catch (Ionic.Zip.ZipException ze) { StringBuilder Msg = new StringBuilder(); Msg.Append("Unable to extract file(s) from archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + ze.Message); switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel)) { case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break; case DialogResult.No: break; case DialogResult.Cancel: throw new CancelException(); default: throw new NotSupportedException(); } } catch (Ionic.Zlib.ZlibException ze) { StringBuilder Msg = new StringBuilder(); Msg.Append("Unable to extract file(s) from archive '" + Archive.ToString() + "'. Do you want to delete this archive so that it will be reconstructed on your next backup?\n\nThe error was: " + ze.Message); switch (MessageBox.Show(Msg.ToString(), "Error", MessageBoxButtons.YesNoCancel)) { case DialogResult.Yes: File.Delete(Project.CompleteBackupFolder + "\\" + Archive.ToString()); break; case DialogResult.No: break; case DialogResult.Cancel: throw new CancelException(); default: throw new NotSupportedException(); } } ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verified " + FilesVerified + " of " + FileList.Count + " files."); if (Continuation.Required) { break; } } if (!MapContinuation.Required) { break; } } string BackupStatusFile = Project.CompleteBackupFolder + "\\Backup_Status.xml"; BackupStatus bs = new BackupStatus(); try { bs = BackupStatus.Load(BackupStatusFile); } catch (Exception) { } bs.LastVerify = DateTime.UtcNow; if (!Continuation.Required) { bs.LastCompletedVerify = DateTime.UtcNow; } if (!Continuation.Required) { bs.LastVerifyRelativePath = ""; } else { bs.LastVerifyRelativePath = Continuation.LastRelativePath; } bs.Save(BackupStatusFile); ZippyForm.LogWriteLine(LogLevel.Information, "Verification complete."); } finally { Directory.Delete(TempRoot.FullName); } } catch (CancelException ce) { ZippyForm.LogWriteLine(LogLevel.Information, "Verification cancelled by user before completion."); throw ce; } catch (Exception ex) { ZippyForm.LogWriteLine(LogLevel.LightDebug, "Verification interrupted by error: " + ex.Message); throw ex; } finally { Progress.Dispose(); Progress = null; System.Diagnostics.Process.GetCurrentProcess().PriorityClass = System.Diagnostics.ProcessPriorityClass.Normal; } }
void RunEntry(ZipFile zip, ArchiveFilename Archive, Manifest.Entry Entry, string DestinationFolder) { if (Entry == null || String.IsNullOrEmpty(DestinationFolder)) { throw new ArgumentException("Invalid manifest entry or processing error for manifest entry.", "Entry"); } bool IsArchiveRoot = (String.IsNullOrEmpty(Entry.Name)) && Entry is Manifest.Folder; Progress.label2.Text = "Extracting: " + Entry.RelativePath; DoEvents(); string Path = Utility.StripTrailingSlash(DestinationFolder) + "\\" + Entry.Name; if (Entry is Manifest.File) { Manifest.File File = (Manifest.File)Entry; try { ArchiveFilename NeededArchive = ArchiveFilename.Parse(File.ArchiveFile); if (NeededArchive == Archive) { Directory.CreateDirectory(DestinationFolder); ExtractFile(zip, File, Path); ExtractProperties(File, Path); BytesCompleted += (long)File.Length; } } catch (CancelException ce) { throw ce; } catch (Exception ex) { throw new Exception(ex.Message + "\nWhile extracting file '" + Entry.RelativePath + "'.", ex); } } else if (Entry is Manifest.Folder) { Manifest.Folder Folder = (Manifest.Folder)Entry; try { Directory.CreateDirectory(Path); foreach (Manifest.File File in Folder.Files) { RunEntry(zip, Archive, File, Path); } foreach (Manifest.Folder Subfolder in Folder.Folders) { RunEntry(zip, Archive, Subfolder, Path); } if (!IsArchiveRoot) { ExtractProperties(Folder, Path); } } catch (CancelException ce) { throw ce; } catch (Exception ex) { throw new Exception(ex.Message + "\nWhile extracting folder '" + Entry.RelativePath + "'.", ex); } } else { throw new ArgumentException(); } Progress.OverallProgressBar.Value = (int)(10000L * BytesCompleted / TotalBytes); return; }
public void OnNewBackup(ArchiveFilename NewArchive) { lock (ArchiveFileList) ArchiveFileList.Archives.Add(NewArchive); }
void WorkerThread() { try { while (!Closing) { Thread.Sleep(0); lock (SearchStateLock) { BackupProject Project = CurrentProject; if (Project == null) { Thread.Sleep(100); IsArchiveNamesComplete = true; IsFolderNamesComplete = true; continue; } string[] SearchWords = CurrentSearchWords; if (SearchWords == null || SearchWords.Length == 0) { // In the case where the user has selected a project but no search details, // we want to show all available backup archives. if (!IsArchiveNamesComplete) { lock (Project.ArchiveFileList) { foreach (ArchiveFilename Backup in Project.ArchiveFileList.Archives) { if (!AllResults.Contains(Backup)) { AllResults.Add(Backup); lock (NewResults) NewResults.Add(Backup); } } } IsArchiveNamesComplete = true; } IsFolderNamesComplete = true; Thread.Sleep(100); continue; } if (!IsArchiveNamesComplete && IncludeArchiveNames) { lock (Project.ArchiveFileList) { foreach (ArchiveFilename Backup in Project.ArchiveFileList.Archives) { string BackupFile = Backup.ToString().ToLower(); bool Match = true; foreach (string Word in SearchWords) { if (!BackupFile.Contains(Word)) { Match = false; break; } } if (!Match) { continue; } if (!AllResults.Contains(Backup)) { AllResults.Add(Backup); lock (NewResults) NewResults.Add(Backup); } } } IsArchiveNamesComplete = true; continue; } if (!IsFolderNamesComplete && IncludeFolderNames) { ArchiveFilename ExamineNext = null; lock (Project.ArchiveFileList) { foreach (ArchiveFilename Backup in Project.ArchiveFileList.Archives) { bool AlreadyDone = false; foreach (ArchiveFilename PosRes in AllResults) { if (Backup == PosRes) { AlreadyDone = true; break; } } if (AlreadyDone) { continue; } foreach (ArchiveFilename NegRes in NegFolderNameResults) { if (Backup == NegRes) { AlreadyDone = true; break; } } if (AlreadyDone) { continue; } ExamineNext = Backup; break; } } if (ExamineNext == null) { IsFolderNamesComplete = true; continue; } /** Examine a single archive on this pass of the loop **/ Manifest Manifest; lock (Project.ManifestCache) { if (!Project.ManifestCache.TryGetValue(ExamineNext, out Manifest)) { try { using (NetworkConnection newconn = new NetworkConnection(Project.CompleteBackupFolder, Project.BackupCredentials)) Manifest = ExamineNext.LoadArchiveManifest(Project, false); Project.ManifestCache.Add(ExamineNext, Manifest); } catch (Ionic.Zip.BadPasswordException) { PasswordBlockedSearch = true; NegFolderNameResults.Add(ExamineNext); continue; } } } if (SearchManifestForFolderName(ExamineNext, Manifest.ArchiveRoot, SearchWords)) { if (!AllResults.Contains(ExamineNext)) { AllResults.Add(ExamineNext); lock (NewResults) NewResults.Add(ExamineNext); } } else { NegFolderNameResults.Add(ExamineNext); } continue; } // If we reach this point, the search has already completed. Idle time. Thread.Sleep(100); } } } catch (Exception ex) { lock (ExceptionLock) { WorkerException = ex; } } }