public BasicResults() { this.BeginTime = DateTime.UtcNow; this.m_parent = null; m_messages = new Library.Utility.FileBackedStringList(); m_warnings = new Library.Utility.FileBackedStringList(); m_errors = new Library.Utility.FileBackedStringList(); m_retryAttempts = new Library.Utility.FileBackedStringList(); m_dbqueue = new Queue <DbMessage>(); m_backendStatistics = new BackendWriter(this); m_callerThread = System.Threading.Thread.CurrentThread; m_backendProgressUpdater = new BackendProgressUpdater(); m_operationProgressUpdater = new OperationProgressUpdater(); }
public BasicResults() { this.BeginTime = DateTime.UtcNow; this.m_parent = null; m_messages = new Library.Utility.FileBackedStringList(); m_warnings = new Library.Utility.FileBackedStringList(); m_errors = new Library.Utility.FileBackedStringList(); m_retryAttempts = new Library.Utility.FileBackedStringList(); m_dbqueue = new Queue<DbMessage>(); m_backendStatistics = new BackendWriter(this); m_callerThread = System.Threading.Thread.CurrentThread; m_backendProgressUpdater = new BackendProgressUpdater(); m_operationProgressUpdater = new OperationProgressUpdater(); }
private bool HandleFilesystemEntry(string path, System.IO.FileAttributes attributes) { // If we lost the connection, there is no point in keeping on processing if (m_backend.HasDied) throw m_backend.LastException; try { m_result.OperationProgressUpdater.StartFile(path, -1); if (m_backendLogFlushTimer < DateTime.Now) { m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN); m_backend.FlushDbMessages(m_database, null); } if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) { if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { m_result.AddVerboseMessage("Ignoring symlink {0}", path); return false; } if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) { Dictionary<string, string> metadata; if (m_options.StoreMetadata) { metadata = m_snapshot.GetMetadata(path); if (metadata == null) metadata = new Dictionary<string, string>(); if (!metadata.ContainsKey("CoreAttributes")) metadata["CoreAttributes"] = attributes.ToString(); if (!metadata.ContainsKey("CoreLastWritetime")) metadata["CoreLastWritetime"] = m_snapshot.GetLastWriteTimeUtc(path).Ticks.ToString(); } else { metadata = new Dictionary<string, string>(); } if (!metadata.ContainsKey("CoreSymlinkTarget")) metadata["CoreSymlinkTarget"] = m_snapshot.GetSymlinkTarget(path); var metahash = Utility.WrapMetadata(metadata, m_options); AddSymlinkToOutput(path, DateTime.UtcNow, metahash); m_result.AddVerboseMessage("Stored symlink {0}", path); //Do not recurse symlinks return false; } } if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) { IMetahash metahash; if (m_options.StoreMetadata) { Dictionary<string, string> metadata = m_snapshot.GetMetadata(path); if (metadata == null) metadata = new Dictionary<string, string>(); if (!metadata.ContainsKey("CoreAttributes")) metadata["CoreAttributes"] = attributes.ToString(); if (!metadata.ContainsKey("CoreLastWritetime")) metadata["CoreLastWritetime"] = m_snapshot.GetLastWriteTimeUtc(path).Ticks.ToString(); metahash = Utility.WrapMetadata(metadata, m_options); } else { metahash = EMPTY_METADATA; } m_result.AddVerboseMessage("Adding directory {0}", path); AddFolderToOutput(path, DateTime.UtcNow, metahash); return true; } m_result.OperationProgressUpdater.UpdatefilesProcessed(++m_result.ExaminedFiles, m_result.SizeOfExaminedFiles); bool changed = false; // The time we scan DateTime scantime = DateTime.UtcNow; // Last scan time DateTime oldScanned; // Last file modification DateTime lastModified = m_snapshot.GetLastWriteTimeUtc(path); var oldId = m_database.GetFileEntry(path, out oldScanned); long filestatsize = m_snapshot.GetFileSize(path); if ((oldId < 0 || m_options.DisableFiletimeCheck || LocalDatabase.NormalizeDateTime(lastModified) >= oldScanned) && (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || filestatsize < m_options.SkipFilesLargerThan)) { m_result.AddVerboseMessage("Checking file for changes {0}", path); m_result.OpenedFiles++; long filesize = 0; IMetahash metahashandsize; if (m_options.StoreMetadata) { Dictionary<string, string> metadata = m_snapshot.GetMetadata(path); if (metadata == null) metadata = new Dictionary<string, string>(); if (!metadata.ContainsKey("CoreAttributes")) metadata["CoreAttributes"] = attributes.ToString(); if (!metadata.ContainsKey("CoreLastWritetime")) metadata["CoreLastWritetime"] = lastModified.Ticks.ToString(); metahashandsize = Utility.WrapMetadata(metadata, m_options); } else { metahashandsize = EMPTY_METADATA; } var hint = m_options.GetCompressionHintFromFilename(path); var oldHash = oldId < 0 ? null : m_database.GetFileHash(oldId); using (var blocklisthashes = new Library.Utility.FileBackedStringList()) using (var hashcollector = new Library.Utility.FileBackedStringList()) { using (var fs = new Blockprocessor(m_snapshot.OpenRead(path), m_blockbuffer)) { try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read file length for file {0}", path), ex); } int blocklistoffset = 0; m_filehasher.Initialize(); var offset = 0; var remaining = fs.Readblock(); do { var size = Math.Min(m_blocksize, remaining); m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset); var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size); if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) { var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); blocklisthashes.Add(blkey); AddBlockToOutput(blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); blocklistoffset = 0; } Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length); blocklistoffset += blockkey.Length; var key = Convert.ToBase64String(blockkey); AddBlockToOutput(key, m_blockbuffer, offset, size, hint, false); hashcollector.Add(key); filesize += size; m_result.OperationProgressUpdater.UpdateFileProgress(filesize); if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return false; remaining -= size; offset += size; if (remaining == 0) { offset = 0; remaining = fs.Readblock(); } } while (remaining > 0); //If all fits in a single block, don't bother with blocklists if (hashcollector.Count > 1) { var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); blocklisthashes.Add(blkeyfinal); AddBlockToOutput(blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); } } m_result.SizeOfOpenedFiles += filesize; m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0); var filekey = Convert.ToBase64String(m_filehasher.Hash); if (oldHash != filekey) { if (oldHash == null) m_result.AddVerboseMessage("New file {0}", path); else m_result.AddVerboseMessage("File has changed {0}", path); if (oldId < 0) { m_result.AddedFiles++; m_result.SizeOfAddedFiles += filesize; if (m_options.Dryrun) m_result.AddDryrunMessage(string.Format("Would add new file {0}, size {1}", path, Library.Utility.Utility.FormatSizeString(filesize))); } else { m_result.ModifiedFiles++; m_result.SizeOfModifiedFiles += filesize; if (m_options.Dryrun) m_result.AddDryrunMessage(string.Format("Would add changed file {0}, size {1}", path, Library.Utility.Utility.FormatSizeString(filesize))); } AddFileToOutput(path, filesize, scantime, metahashandsize, hashcollector, filekey, blocklisthashes); changed = true; } else { // When we write the file to output, update the scan time oldScanned = scantime; m_result.AddVerboseMessage("File has not changed {0}", path); } } } else { if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || m_snapshot.GetFileSize(path) < m_options.SkipFilesLargerThan) m_result.AddVerboseMessage("Skipped checking file, because timestamp was not updated {0}", path); else m_result.AddVerboseMessage("Skipped checking file, because the size exceeds limit {0}", path); } if (!changed) AddUnmodifiedFile(oldId, oldScanned); m_result.SizeOfExaminedFiles += filestatsize; if (filestatsize != 0) m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to process path: {0}", path), ex); m_result.FilesWithError++; } return true; }
public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader) { return(AutomationExtensions.RunTask( new { Input = Channels.StreamBlock.ForRead, ProgressChannel = Channels.ProgressEvents.ForWrite, BlockOutput = Channels.OutputBlocks.ForWrite }, async self => { var blocksize = options.Blocksize; var filehasher = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.FileHashAlgorithm); var blockhasher = Duplicati.Library.Utility.HashAlgorithmHelper.Create(options.BlockHashAlgorithm); var emptymetadata = Utility.WrapMetadata(new Dictionary <string, string>(), options); var maxmetadatasize = (options.Blocksize / (long)options.BlockhashSize) * options.Blocksize; if (blockhasher == null) { throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported"); } if (filehasher == null) { throw new UserInformationException(Strings.Common.InvalidHashAlgorithm(options.FileHashAlgorithm), "FileHashAlgorithmNotSupported"); } if (!blockhasher.CanReuseTransform) { throw new UserInformationException(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm), "BlockHashAlgorithmNotSupported"); } if (!filehasher.CanReuseTransform) { throw new UserInformationException(Strings.Common.InvalidCryptoSystem(options.FileHashAlgorithm), "FileHashAlgorithmNotSupported"); } using (var empty_metadata_stream = new MemoryStream(emptymetadata.Blob)) while (await taskreader.ProgressAsync) { var send_close = false; var filesize = 0L; var filename = string.Empty; var e = await self.Input.ReadAsync(); var cur = e.Result; try { var stream = e.Stream; using (var blocklisthashes = new Library.Utility.FileBackedStringList()) using (var hashcollector = new Library.Utility.FileBackedStringList()) { var blocklistbuffer = new byte[blocksize]; var blocklistoffset = 0L; long fslen = -1; try { fslen = stream.Length; } catch (Exception ex) { Logging.Log.WriteWarningMessage(FILELOGTAG, "FileLengthFailure", ex, "Failed to read file length for file {0}", e.Path); } if (e.IsMetadata && fslen > maxmetadatasize) { //TODO: To fix this, the "WriteFileset" method in BackupHandler needs to // be updated such that it can select sets even when there are multiple // blocklist hashes for the metadata. // This could be done such that an extra query is made if the metadata // spans multiple blocklist hashes, as it is not expected to be common Logging.Log.WriteWarningMessage(LOGTAG, "TooLargeMetadata", null, "Metadata size is {0}, but the largest accepted size is {1}, recording empty metadata for {2}", fslen, maxmetadatasize, e.Path); empty_metadata_stream.Position = 0; stream = empty_metadata_stream; fslen = stream.Length; } await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = fslen, Type = EventType.FileStarted }); send_close = true; filehasher.Initialize(); var lastread = 0; var buf = new byte[blocksize]; var lastupdate = DateTime.Now; // Core processing loop, read blocks of data and hash individually while (((lastread = await stream.ForceStreamReadAsync(buf, blocksize)) != 0)) { // Run file hashing concurrently to squeeze a little extra concurrency out of it var pftask = Task.Run(() => filehasher.TransformBlock(buf, 0, lastread, buf, 0)); var hashdata = blockhasher.ComputeHash(buf, 0, lastread); var hashkey = Convert.ToBase64String(hashdata); // If we have too many hashes, flush the blocklist if (blocklistbuffer.Length - blocklistoffset < hashdata.Length) { var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, (int)blocklistoffset)); blocklisthashes.Add(blkey); await DataBlock.AddBlockToOutputAsync(self.BlockOutput, blkey, blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); blocklistoffset = 0; blocklistbuffer = new byte[blocksize]; } // Store the current hash in the blocklist Array.Copy(hashdata, 0, blocklistbuffer, blocklistoffset, hashdata.Length); blocklistoffset += hashdata.Length; hashcollector.Add(hashkey); filesize += lastread; // Don't spam updates if ((DateTime.Now - lastupdate).TotalSeconds > 10) { await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filesize, Type = EventType.FileProgressUpdate }); lastupdate = DateTime.Now; } // Make sure the filehasher is done with the buf instance before we pass it on await pftask; await DataBlock.AddBlockToOutputAsync(self.BlockOutput, hashkey, buf, 0, lastread, e.Hint, true); buf = new byte[blocksize]; } // If we have more than a single block of data, output the (trailing) blocklist if (hashcollector.Count > 1) { var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, (int)blocklistoffset)); blocklisthashes.Add(blkey); await DataBlock.AddBlockToOutputAsync(self.BlockOutput, blkey, blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); } filehasher.TransformFinalBlock(new byte[0], 0, 0); var filehash = Convert.ToBase64String(filehasher.Hash); var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes); cur.SetResult(new StreamProcessResult() { Streamlength = filesize, Streamhash = filehash, Blocksetid = blocksetid }); cur = null; } } catch (Exception ex) { try { if (cur != null) { cur.TrySetException(ex); } } catch { } // Rethrow if (ex.IsRetiredException()) { throw; } } finally { if (cur != null) { try { cur.TrySetCanceled(); } catch { } cur = null; } if (send_close) { await self.ProgressChannel.WriteAsync(new ProgressEvent() { Filepath = e.Path, Length = filesize, Type = EventType.FileClosed }); } send_close = false; } } })); }