private bool HandleFilesystemEntry(Snapshots.ISnapshotService snapshot, BackendManager backend, string path, System.IO.FileAttributes attributes) { // If we lost the connection, there is no point in keeping on processing if (backend.HasDied) throw backend.LastException; try { m_result.OperationProgressUpdater.StartFile(path, -1); if (m_backendLogFlushTimer < DateTime.Now) { m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN); backend.FlushDbMessages(m_database, null); } DateTime lastwrite = new DateTime(0, DateTimeKind.Utc); try { lastwrite = snapshot.GetLastWriteTimeUtc(path); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read timestamp on \"{0}\"", path), ex); } if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) { if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { m_result.AddVerboseMessage("Ignoring symlink {0}", path); return false; } if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) { Dictionary<string, string> metadata = GenerateMetadata(snapshot, path, attributes); if (!metadata.ContainsKey("CoreSymlinkTarget")) { var p = snapshot.GetSymlinkTarget(path); if (string.IsNullOrWhiteSpace(p)) m_result.AddVerboseMessage("Ignoring empty symlink {0}", path); else metadata["CoreSymlinkTarget"] = p; } var metahash = Utility.WrapMetadata(metadata, m_options); AddSymlinkToOutput(backend, path, DateTime.UtcNow, metahash); m_result.AddVerboseMessage("Stored symlink {0}", path); //Do not recurse symlinks return false; } } if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) { IMetahash metahash; if (m_options.StoreMetadata) { metahash = Utility.WrapMetadata(GenerateMetadata(snapshot, path, attributes), m_options); } else { metahash = EMPTY_METADATA; } m_result.AddVerboseMessage("Adding directory {0}", path); AddFolderToOutput(backend, path, lastwrite, metahash); return true; } m_result.OperationProgressUpdater.UpdatefilesProcessed(++m_result.ExaminedFiles, m_result.SizeOfExaminedFiles); bool changed = false; // Last scan time DateTime oldModified; long lastFileSize = -1; string oldMetahash; long oldMetasize; var oldId = m_database.GetFileEntry(path, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); long filestatsize = -1; try { filestatsize = snapshot.GetFileSize(path); } catch { } IMetahash metahashandsize = m_options.StoreMetadata ? Utility.WrapMetadata(GenerateMetadata(snapshot, path, attributes), m_options) : EMPTY_METADATA; var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0; var filesizeChanged = filestatsize < 0 || lastFileSize < 0 || filestatsize != lastFileSize; var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan; var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Size != oldMetasize || metahashandsize.Hash != oldMetahash); if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged || metadatachanged) && !tooLargeFile) { m_result.AddVerboseMessage("Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", path, oldId <= 0, timestampChanged, filesizeChanged, metadatachanged, lastwrite, oldModified); m_result.OpenedFiles++; long filesize = 0; var hint = m_options.GetCompressionHintFromFilename(path); var oldHash = oldId < 0 ? null : m_database.GetFileHash(oldId); using (var blocklisthashes = new Library.Utility.FileBackedStringList()) using (var hashcollector = new Library.Utility.FileBackedStringList()) { using (var fs = new Blockprocessor(snapshot.OpenRead(path), m_blockbuffer)) { try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read file length for file {0}", path), ex); } int blocklistoffset = 0; m_filehasher.Initialize(); var offset = 0; var remaining = fs.Readblock(); do { var size = Math.Min(m_blocksize, remaining); m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset); var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size); if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) { var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); blocklisthashes.Add(blkey); AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); blocklistoffset = 0; } Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length); blocklistoffset += blockkey.Length; var key = Convert.ToBase64String(blockkey); AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false); hashcollector.Add(key); filesize += size; m_result.OperationProgressUpdater.UpdateFileProgress(filesize); if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return false; remaining -= size; offset += size; if (remaining == 0) { offset = 0; remaining = fs.Readblock(); } } while (remaining > 0); //If all fits in a single block, don't bother with blocklists if (hashcollector.Count > 1) { var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); blocklisthashes.Add(blkeyfinal); AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); } } m_result.SizeOfOpenedFiles += filesize; m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0); var filekey = Convert.ToBase64String(m_filehasher.Hash); if (oldHash != filekey) { if (oldHash == null) m_result.AddVerboseMessage("New file {0}", path); else m_result.AddVerboseMessage("File has changed {0}", path); if (oldId < 0) { m_result.AddedFiles++; m_result.SizeOfAddedFiles += filesize; if (m_options.Dryrun) m_result.AddDryrunMessage(string.Format("Would add new file {0}, size {1}", path, Library.Utility.Utility.FormatSizeString(filesize))); } else { m_result.ModifiedFiles++; m_result.SizeOfModifiedFiles += filesize; if (m_options.Dryrun) m_result.AddDryrunMessage(string.Format("Would add changed file {0}, size {1}", path, Library.Utility.Utility.FormatSizeString(filesize))); } AddFileToOutput(backend, path, filesize, lastwrite, metahashandsize, hashcollector, filekey, blocklisthashes); changed = true; } else if (metadatachanged) { m_result.AddVerboseMessage("File has only metadata changes {0}", path); AddFileToOutput(backend, path, filesize, lastwrite, metahashandsize, hashcollector, filekey, blocklisthashes); changed = true; } else { // When we write the file to output, update the last modified time oldModified = lastwrite; m_result.AddVerboseMessage("File has not changed {0}", path); } } } else { if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || snapshot.GetFileSize(path) < m_options.SkipFilesLargerThan) m_result.AddVerboseMessage("Skipped checking file, because timestamp was not updated {0}", path); else m_result.AddVerboseMessage("Skipped checking file, because the size exceeds limit {0}", path); } if (!changed) AddUnmodifiedFile(oldId, lastwrite); m_result.SizeOfExaminedFiles += filestatsize; if (filestatsize != 0) m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to process path: {0}", path), ex); m_result.FilesWithError++; } return true; }
private Dictionary<string, string> GenerateMetadata(Snapshots.ISnapshotService snapshot, string path, System.IO.FileAttributes attributes) { try { Dictionary<string, string> metadata; if (m_options.StoreMetadata) { metadata = snapshot.GetMetadata(path); if (metadata == null) metadata = new Dictionary<string, string>(); if (!metadata.ContainsKey("CoreAttributes")) metadata["CoreAttributes"] = attributes.ToString(); if (!metadata.ContainsKey("CoreLastWritetime")) { try { metadata["CoreLastWritetime"] = snapshot.GetLastWriteTimeUtc(path).Ticks.ToString(); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read timestamp on \"{0}\"", path), ex); } } if (!metadata.ContainsKey("CoreCreatetime")) { try { metadata["CoreCreatetime"] = snapshot.GetCreationTimeUtc(path).Ticks.ToString(); } catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read timestamp on \"{0}\"", path), ex); } } } else { metadata = new Dictionary<string, string>(); } return metadata; } catch(Exception ex) { m_result.AddWarning(string.Format("Failed to process metadata for \"{0}\", storing empty metadata", path), ex); return new Dictionary<string, string>(); } }