Beispiel #1
0
        public void Bind(ArchiveReader.Stats stats)
        {
            onTick = delegate {
                this.Text = stats.Title;
                progressBar1.Value = Math.Max(0, Math.Min(100, (int)(stats.Progress * 100)));
                textStatus.Text = stats.Status;

                TimeSpan time = (stats.EndTime ?? DateTime.Now) - stats.StartTime;
                TimeSpan writeTime = (stats.EndTime ?? DateTime.Now) - (stats.WriteStartTime ?? DateTime.Now);
                long writeSpeed = (int)writeTime.TotalSeconds == 0 ? 0 : stats.TotalWritten / (int)writeTime.TotalSeconds;
                labelL1.Text = "Total processed:";          textL1.Text = FormatSize(stats.TotalWritten + stats.Unmodified);
                labelL2.Text = "Elapsed time:";             textL2.Text = string.Format("{0}:{1:D2}:{2:D2}", time.Hours, time.Minutes, time.Seconds);
                labelL3.Text = "Average speed:";            textL3.Text = writeSpeed == 0 ? "-" : (FormatSize(writeSpeed) + "/s");
                labelL4.Text = "";                          textL4.Text = "";
                labelR1.Text = "Unmodified:";               textR1.Text = FormatSize(stats.Unmodified,                stats.Unmodified + stats.ReadFromArchiveCompressed + stats.ReadFromWorkingCopy);
                labelR2.Text = "Read from archive:";        textR2.Text = FormatSize(stats.ReadFromArchiveCompressed, stats.Unmodified + stats.ReadFromArchiveCompressed + stats.ReadFromWorkingCopy);
                labelR3.Text = "Read from working copy:";   textR3.Text = FormatSize(stats.ReadFromWorkingCopy,       stats.Unmodified + stats.ReadFromArchiveCompressed + stats.ReadFromWorkingCopy);
                labelR4.Text = "";                          textR4.Text = "";

                buttonCancel.Enabled = (stats.EndTime == null);
            };
            onCancel = delegate {
                stats.Canceled = true;
                stats.EndTime  = DateTime.Now;
            };
        }
Beispiel #2
0
        public static bool Extract(string archiveFilename, string destination, Stats stats)
        {
            stats.Title = Path.GetFileName(archiveFilename);

            ArchiveReader archive = new ArchiveReader(archiveFilename, stats);

            bool writeEnabled = (destination != null);

            Dictionary<string, ZipFile> openZips = new Dictionary<string, ZipFile>();
            openZips[archive.ArchiveName.ToLowerInvariant()] = archive.zipFile;

            FileStream   openFile = null;
            IAsyncResult openFileRead = null;
            string       openFilePathLC = null;

            long totalSize = 0;
            long totalSizeDone = 0;

            // Setup cache
            Dictionary<string, ExtractedData> dataCache = new Dictionary<string, ExtractedData>();
            int time = 0;
            foreach (File file in archive.files) {
                totalSize += file.Size;
                foreach (int hashIndex in file.HashIndices) {
                    if (stats.Canceled) return false;

                    string path = archive.GetString(archive.hashes[hashIndex].Path).ToLowerInvariant();
                    if (!dataCache.ContainsKey(path)) dataCache.Add(path, new ExtractedData());
                    dataCache[path].Refs.Add(time++);
                }
            }

            string stateFile = writeEnabled ? Path.Combine(destination, Settings.StateFile) : null;

            stats.Status = "Loading working copy state";
            WorkingCopy workingCopy = writeEnabled ? WorkingCopy.Load(stateFile) : new WorkingCopy();
            WorkingCopy newWorkingFiles = new WorkingCopy();
            List<WorkingHash> oldWorkingHashes = new List<WorkingHash>();
            if (writeEnabled) {
                int oldCount = workingCopy.Count;
                workingCopy = WorkingCopy.HashLocalFiles(destination, stats, workingCopy);
                if (workingCopy.Count > oldCount) {
                    stats.Status = "Saving working copy state";
                    workingCopy.Save(stateFile);
                }
            }
            foreach(WorkingFile wf in workingCopy.GetAll()) {
                foreach(WorkingHash wh in wf.Hashes) {
                    wh.File = wf;
                }
                oldWorkingHashes.AddRange(wf.Hashes);
            }
            oldWorkingHashes.Sort();
            if (stats.Canceled) return false;

            string tmpPath = null;
            if (writeEnabled) {
                tmpPath = Path.Combine(destination, Settings.TmpDirectory);
                Directory.CreateDirectory(tmpPath);
            }

            Dictionary<ExtractedData, bool> loaded = new Dictionary<ExtractedData, bool>();
            float waitingForDecompression = 0.0f;
            float mbUnloadedDueToMemoryPressure = 0.0f;

            stats.Status = writeEnabled ? "Extracting" : "Verifying";
            stats.WriteStartTime = DateTime.Now;

            foreach (File file in archive.files) {

                string tmpFileName = null;
                FileStream outFile = null;

                if (writeEnabled) {
                    // Quickpath - see if the file exists and has correct content
                    WorkingFile workingFile = workingCopy.Find(Path.Combine(destination, file.Name));
                    if (workingFile != null && workingFile.ExistsOnDisk() && !workingFile.IsModifiedOnDisk()) {
                        if (new Hash(workingFile.Hash).CompareTo(new Hash(file.Hash)) == 0) {
                            // The file is already there - no need to extract it
                            stats.Status = "Skipped " + file.Name;
                            workingFile.UserModified = false;
                            newWorkingFiles.Add(workingFile);
                            stats.Unmodified += file.Size;
                            totalSizeDone += file.Size;
                            continue;
                        }
                    }

                    int tmpFileNamePostfix = 0;
                    do {
                        tmpFileName = Path.Combine(tmpPath, file.Name + (tmpFileNamePostfix == 0 ? string.Empty : ("-" + tmpFileNamePostfix.ToString())));
                        tmpFileNamePostfix++;
                    } while (System.IO.File.Exists(tmpFileName));

                    Directory.CreateDirectory(Path.GetDirectoryName(tmpFileName));
                    outFile = new FileStream(tmpFileName, FileMode.CreateNew, FileAccess.Write);
                    // Avoid fragmentation
                    outFile.SetLength(file.Size);
                    outFile.Position = 0;
                }

                List<WorkingHash> workingHashes = new List<WorkingHash>();

                try {
                    stats.Progress = 0;
                    stats.Status = (writeEnabled ? "Extracting " : "Verifying ") + file.Name;

                    SHA1CryptoServiceProvider sha1Provider = new SHA1CryptoServiceProvider();

                    Queue<MemoryStreamRef> writeQueue = new Queue<MemoryStreamRef>();

                    int p = 0;
                    for (int i = 0; i < file.HashIndices.Count; i++) {
                        if (stats.Canceled) {
                            stats.Status = "Canceled.  No files were modified.";
                            return false;
                        }

                        // Prefetch
                        for (; p < file.HashIndices.Count; p++) {
                            if (writeQueue.Count > 0 && writeQueue.Peek().Ready.WaitOne(TimeSpan.Zero)) break; // Some data is ready - go process it

                            int prefetchSize = 0;
                            Dictionary<MemoryStream, bool> prefetchedStreams = new Dictionary<MemoryStream, bool>();
                            foreach(MemoryStreamRef memStreamRef in writeQueue) {
                                prefetchedStreams[memStreamRef.MemStream] = true;
                            }
                            foreach(MemoryStream prefetchedStream in prefetchedStreams.Keys) {
                                prefetchSize += (int)prefetchedStream.Length;
                            }
                            if (writeQueue.Count > 0 && prefetchSize > Settings.WritePrefetchSize) break;  // We have prefetched enough data

                            HashSource hashSrc = archive.hashes[file.HashIndices[p]];
                            string path = archive.GetString(hashSrc.Path).ToLowerInvariant();
                            ExtractedData data = dataCache[path];

                            // See if we have the hash on disk.  Try our best not to seek too much
                            WorkingHash onDiskHash = null;
                            long bestSeekDistance = long.MaxValue;
                            int idx = oldWorkingHashes.BinarySearch(new WorkingHash() { Hash = hashSrc.Hash });
                            if (idx >= 0) {
                                while (idx - 1 >= 0 && oldWorkingHashes[idx - 1].Hash.Equals(hashSrc.Hash)) idx--;
                                for (; idx < oldWorkingHashes.Count && oldWorkingHashes[idx].Hash.Equals(hashSrc.Hash); idx++) {
                                    WorkingHash wh = oldWorkingHashes[idx];
                                    long seekDistance;
                                    if (openFile != null && openFilePathLC == wh.File.NameLowercase) {
                                        seekDistance = Math.Abs(openFile.Position - wh.Offset);
                                    } else {
                                        seekDistance = long.MaxValue;
                                    }
                                    if (onDiskHash == null || seekDistance < bestSeekDistance) {
                                        onDiskHash = wh;
                                        bestSeekDistance = seekDistance;
                                    }
                                }
                            }

                            if (onDiskHash != null && ((openFilePathLC == onDiskHash.File.NameLowercase) || (onDiskHash.File.ExistsOnDisk() && !onDiskHash.File.IsModifiedOnDisk()))) {
                                MemoryStream memStream = new MemoryStream(onDiskHash.Length);
                                memStream.SetLength(onDiskHash.Length);
                                // Finish the last read
                                if (openFileRead != null) {
                                    openFile.EndRead(openFileRead);
                                    openFileRead = null;
                                }
                                // Open other file
                                if (openFilePathLC != onDiskHash.File.NameLowercase) {
                                    if (openFile != null) openFile.Close();
                                    openFile = new FileStream(onDiskHash.File.NameLowercase, FileMode.Open, FileAccess.Read, FileShare.Read, Settings.FileStreamBufferSize, FileOptions.None);
                                    openFilePathLC = onDiskHash.File.NameLowercase;
                                    System.Diagnostics.Debug.Write(Path.GetFileName(onDiskHash.File.NameMixedcase));
                                }
                                System.Diagnostics.Debug.Write(onDiskHash.Offset == openFile.Position ? "." : "S");
                                if (openFile.Position != onDiskHash.Offset)
                                    openFile.Position = onDiskHash.Offset;
                                openFileRead = openFile.BeginRead(memStream.GetBuffer(), 0, (int)memStream.Length, null, null);
                                writeQueue.Enqueue(new MemoryStreamRef() {
                                    Ready     = openFileRead.AsyncWaitHandle,
                                    MemStream = memStream,
                                    Offset    = 0,
                                    Length    = (int)memStream.Length,
                                    CacheLine = null,
                                    Hash      = hashSrc.Hash
                                });
                                stats.ReadFromWorkingCopy += hashSrc.Length;
                            } else {
                                // Locate and load the zipentry
                                ZipEntry pZipEntry;
                                path = path.Replace("\\", "/");
                                if (path.StartsWith("/")) {
                                    pZipEntry = archive.zipFile[path.Substring(1)];
                                } else {
                                    int slashIndex = path.IndexOf("/");
                                    string zipPath = path.Substring(0, slashIndex);
                                    string entryPath = path.Substring(slashIndex + 1);
                                    if (!openZips.ContainsKey(zipPath)) openZips[zipPath] = new ZipFile(Path.Combine(Path.GetDirectoryName(archiveFilename), zipPath));
                                    pZipEntry = openZips[zipPath][entryPath];
                                }

                                if (data.Data == null) {
                                    stats.ReadFromArchiveDecompressed += pZipEntry.UncompressedSize;
                                    stats.ReadFromArchiveCompressed   += pZipEntry.CompressedSize;
                                    data.AsycDecompress(pZipEntry);
                                }
                                loaded[data] = true;

                                writeQueue.Enqueue(new MemoryStreamRef() {
                                    Ready     = data.LoadDone,
                                    MemStream = data.Data,
                                    Offset    = hashSrc.Offset,
                                    Length    = hashSrc.Length,
                                    CacheLine = data,
                                    Hash      = hashSrc.Hash
                                });
                            }
                        }

                        MemoryStreamRef writeItem = writeQueue.Dequeue();

                        while (writeItem.Ready.WaitOne(TimeSpan.FromSeconds(0.01)) == false) {
                            waitingForDecompression += 0.01f;
                        }

                        // Write output
                        if (writeEnabled) {
                            workingHashes.Add(new WorkingHash() {
                                Hash = writeItem.Hash,
                                Offset = outFile.Position,
                                Length = writeItem.Length
                            });

                            outFile.Write(writeItem.MemStream.GetBuffer(), (int)writeItem.Offset, writeItem.Length);
                        }

                        // Verify SHA1
                        sha1Provider.TransformBlock(writeItem.MemStream.GetBuffer(), (int)writeItem.Offset, writeItem.Length, writeItem.MemStream.GetBuffer(), (int)writeItem.Offset);

                        stats.TotalWritten += writeItem.Length;
                        totalSizeDone += writeItem.Length;

                        stats.Title = string.Format("{0:F0}% {1}", 100 * (float)totalSizeDone / (float)totalSize , Path.GetFileName(archiveFilename));
                        stats.Progress = (float)i / (float)file.HashIndices.Count;

                        // Unload if it is not needed anymore
                        if (writeItem.CacheLine != null) {
                            writeItem.CacheLine.Refs.RemoveAt(0);
                            if (writeItem.CacheLine.Refs.Count == 0) {
                                StreamPool.Release(ref writeItem.CacheLine.Data);
                                writeItem.CacheLine.LoadDone = null;
                                loaded.Remove(writeItem.CacheLine);
                            }
                        }

                        // Unload some data if we are running out of memory
                        while (loaded.Count * Settings.MaxZipEntrySize > Settings.WriteCacheSize) {
                            ExtractedData maxRef = null;
                            foreach (ExtractedData ed in loaded.Keys) {
                                if (maxRef == null || ed.Refs[0] > maxRef.Refs[0]) maxRef = ed;
                            }
                            maxRef.LoadDone.WaitOne();

                            // Check that we are not evicting something from the write queue
                            bool inQueue = false;
                            foreach(MemoryStreamRef memRef in writeQueue) {
                                if (memRef.CacheLine == maxRef) inQueue = true;
                            }
                            if (inQueue) break;

                            mbUnloadedDueToMemoryPressure += (float)maxRef.Data.Length / 1024 / 1024;
                            StreamPool.Release(ref maxRef.Data);
                            maxRef.LoadDone = null;
                            loaded.Remove(maxRef);
                        }
                    }
                    stats.Progress = 0;

                    sha1Provider.TransformFinalBlock(new byte[0], 0, 0);
                    byte[] sha1 = sha1Provider.Hash;

                    if (new Hash(sha1).CompareTo(new Hash(file.Hash)) != 0) {
                        MessageBox.Show("The checksum of " + file.Name + " does not match original value.  The file is corrupted.", "Critical error", MessageBoxButtons.OK, MessageBoxIcon.Error);
                        if (writeEnabled) {
                            stats.Status = "Extraction failed.  Checksum mismatch.";
                        } else {
                            stats.Status = "Verification failed.  Checksum mismatch.";
                        }
                        return false;
                    }

                } finally {
                    if (outFile != null) outFile.Close();
                }

                if (writeEnabled) {
                    FileInfo fileInfo = new FileInfo(tmpFileName);

                    WorkingFile workingFile = new WorkingFile() {
                        NameMixedcase = Path.Combine(destination, file.Name),
                        Size = fileInfo.Length,
                        Created = fileInfo.CreationTime,
                        Modified = fileInfo.LastWriteTime,
                        Hash = file.Hash,
                        TempFileName = tmpFileName,
                        Hashes = workingHashes
                    };
                    newWorkingFiles.Add(workingFile);
                }
            }

            stats.Progress = 0;
            stats.Title = string.Format("100% {0}", Path.GetFileName(archiveFilename));

            // Close sources
            foreach (ZipFile zip in openZips.Values) {
                zip.Dispose();
            }
            if (openFileRead != null) openFile.EndRead(openFileRead);
            if (openFile != null)     openFile.Close();

            // Replace the old working copy with new one
            if (writeEnabled) {
                List<string> deleteFilesLC = new List<string>();
                List<string> deleteFilesAskLC = new List<string>();
                List<string> keepFilesLC = new List<string>();

                stats.Status = "Preparing to move files";

                // Delete all non-user-modified files
                foreach (WorkingFile workingFile in workingCopy.GetAll()) {
                    if (!workingFile.UserModified && workingFile.ExistsOnDisk() && !workingFile.IsModifiedOnDisk()) {
                        WorkingFile newWF = newWorkingFiles.Find(workingFile.NameLowercase);
                        // Do not delete if it is was skipped 'fast-path' file
                        if (newWF != null && newWF.TempFileName == null) continue;
                        deleteFilesLC.Add(workingFile.NameLowercase);
                    }
                }

                // Find obstructions for new files
                foreach (WorkingFile newWorkingFile in newWorkingFiles.GetAll()) {
                    if (newWorkingFile.TempFileName != null && newWorkingFile.ExistsOnDisk() && !deleteFilesLC.Contains(newWorkingFile.NameLowercase)) {
                        deleteFilesAskLC.Add(newWorkingFile.NameLowercase);
                    }
                }

                // Ask the user for permission to delete
                StringBuilder sb = new StringBuilder();
                sb.AppendLine("Do you want to override local changes in the following files?");
                int numLines = 0;
                foreach (string deleteFileAskLC in deleteFilesAskLC) {
                    sb.AppendLine(deleteFileAskLC);
                    numLines++;
                    if (numLines > 30) {
                        sb.AppendLine("...");
                        sb.AppendLine("(" + deleteFilesAskLC.Count + " files in total)");
                        break;
                    }
                }
                if (deleteFilesAskLC.Count > 0) {
                    DialogResult overrideAnswer = Settings.AlwaysOverwrite ? DialogResult.Yes : MessageBox.Show(sb.ToString(), "Override files", MessageBoxButtons.YesNoCancel);
                    if (overrideAnswer == DialogResult.Cancel) {
                        stats.Status = "Canceled.  No files were modified.";
                        return false;
                    }
                    if (overrideAnswer == DialogResult.Yes) {
                        deleteFilesLC.AddRange(deleteFilesAskLC);
                    } else {
                        keepFilesLC = deleteFilesAskLC;
                    }
                    deleteFilesAskLC.Clear();
                }

                // Delete files
                foreach (string deleteFileLC in deleteFilesLC) {
                    stats.Status = "Deleting " + Path.GetFileName(deleteFileLC);
                    while (true) {
                        try {
                            FileInfo fileInfo = new FileInfo(deleteFileLC);
                            if (fileInfo.IsReadOnly) {
                                fileInfo.IsReadOnly = false;
                            }
                            System.IO.File.Delete(deleteFileLC);
                            workingCopy.Remove(deleteFileLC);
                            break;
                        } catch (Exception e) {
                            DialogResult deleteAnswer = MessageBox.Show("Can not delete file " + deleteFileLC + Environment.NewLine + e.Message, "Error", MessageBoxButtons.AbortRetryIgnore);
                            if (deleteAnswer == DialogResult.Retry) continue;
                            if (deleteAnswer == DialogResult.Ignore) break;
                            if (deleteAnswer == DialogResult.Abort) {
                                stats.Status = "Canceled.  Some files were deleted.";
                                return false;
                            }
                        }
                    }
                }

                // Move the new files
                foreach (WorkingFile newWorkingFile in newWorkingFiles.GetAll()) {
                    if (!keepFilesLC.Contains(newWorkingFile.NameLowercase) && newWorkingFile.TempFileName != null) {
                        stats.Status = "Moving " + Path.GetFileName(newWorkingFile.NameMixedcase);
                        while (true) {
                            try {
                                Directory.CreateDirectory(Path.GetDirectoryName(newWorkingFile.NameMixedcase));
                                System.IO.File.Move(newWorkingFile.TempFileName, newWorkingFile.NameMixedcase);
                                workingCopy.Add(newWorkingFile);
                                break;
                            } catch (Exception e) {
                                DialogResult moveAnswer = MessageBox.Show("Error when moving " + newWorkingFile.TempFileName + Environment.NewLine + e.Message, "Error", MessageBoxButtons.AbortRetryIgnore);
                                if (moveAnswer == DialogResult.Retry) continue;
                                if (moveAnswer == DialogResult.Ignore) break;
                                if (moveAnswer == DialogResult.Abort) {
                                    stats.Status = "Canceled.  Some files were deleted or overridden.";
                                    return false;
                                }
                            }
                        }
                    }
                }

                stats.Status = "Saving working copy state";
                workingCopy.Save(stateFile);

                stats.Status = "Deleting temporary directory";
                try {
                    if (Directory.Exists(tmpPath)) Directory.Delete(tmpPath, true);
                } catch {
                }
            }

            stats.EndTime = DateTime.Now;
            if (writeEnabled) {
                stats.Status = "Extraction finished";
            } else {
                stats.Status = "Verification finished";
            }
            return true;
        }