public static UpdateCacheEntry ReadFromStream(BinaryReader reader) { var fileEntry = new UpdateCacheEntry { FilePath = reader.ReadString(), LastWriteTimeUtcTicks = reader.ReadInt64(), FileSize = reader.ReadInt64(), }; return(fileEntry); }
private void DownloadAndCalculateChangedFiles(ConcurrentDictionary <string, FileInfo> localFileCache, StringBuilder fileListToDelete, ConcurrentDictionary <string, bool> filesNoUpdateNeeded) { try { using (MemoryStream currentRemoteCacheFile = new MemoryStream()) { DownloadFile(_uploadCacheFileName, currentRemoteCacheFile); PrintTime($"Remote file cache '{_uploadCacheFileName}' downloaded"); currentRemoteCacheFile.Seek(0, SeekOrigin.Begin); using (BinaryReader reader = new BinaryReader(currentRemoteCacheFile)) { int entryCount = reader.ReadInt32(); int deleteFileCount = 0; long deleteFileSize = 0; int upToDateFileCount = 0; long upToDateFileSize = 0; long remoteFilesSize = 0; for (int i = 0; i < entryCount; i++) { var remotefileEntry = UpdateCacheEntry.ReadFromStream(reader); remoteFilesSize += remotefileEntry.FileSize; FileInfo localFileInfo; if (!localFileCache.TryGetValue(remotefileEntry.FilePath, out localFileInfo)) { deleteFileCount++; deleteFileSize += remotefileEntry.FileSize; fileListToDelete.Append(remotefileEntry.FilePath).Append("\n"); } else if (localFileInfo.LastWriteTimeUtc.Ticks == remotefileEntry.LastWriteTimeUtcTicks && localFileInfo.Length == remotefileEntry.FileSize) { upToDateFileCount++; upToDateFileSize += remotefileEntry.FileSize; filesNoUpdateNeeded[remotefileEntry.FilePath] = true; } } PrintTime($"{deleteFileCount,7:n0} [{deleteFileSize,13:n0} bytes] of {entryCount,7:n0} [{remoteFilesSize,13:n0} bytes] files need to be deleted"); PrintTime($"{upToDateFileCount,7:n0} [{upToDateFileSize,13:n0} bytes] of {entryCount,7:n0} [{remoteFilesSize,13:n0} bytes] files don't need to be updated"); } } } catch (Exception ex) { // TODO log verbose messages differently var msg = ex.Message; PrintTime($"Remote file cache '{_uploadCacheFileName}' not found! We are uploading all files!"); } PrintTime($"Diff between local and remote file cache calculated"); }
private UpdateFlages CreateAndUploadFileDiff(ConcurrentDictionary <string, FileInfo> localFileCache, ConcurrentDictionary <string, bool> filesNoUpdateNeeded, string fileListToDelete) { var updateFlags = UpdateFlages.NONE; using (Stream tarGzStream = new MemoryStream()) { using (var tarGzWriter = WriterFactory.Open(tarGzStream, ArchiveType.Tar, CompressionType.GZip)) { using (MemoryStream newCacheFileStream = new MemoryStream()) { using (BinaryWriter newCacheFileWriter = new BinaryWriter(newCacheFileStream)) { newCacheFileWriter.Write(localFileCache.Count); var updateNeeded = false; var updateFileCount = 0; long updateFileSize = 0; var allFileCount = 0; long allFileSize = 0; foreach (var file in localFileCache) { allFileCount++; allFileSize += file.Value.Length; // add new cache file entry UpdateCacheEntry.WriteToStream(newCacheFileWriter, file.Key, file.Value); // add new file entry if (filesNoUpdateNeeded.ContainsKey(file.Key)) { continue; } updateNeeded = true; updateFileCount++; updateFileSize += file.Value.Length; try { tarGzWriter.Write(file.Key, file.Value); } catch (IOException ioEx) { PrintError(ioEx, withStacktrace: false); } catch (Exception ex) { PrintError(ex); } } PrintTime($"{updateFileCount,7:n0} [{updateFileSize,13:n0} bytes] of {allFileCount,7:n0} [{allFileSize,13:n0} bytes] files need to be updated"); if (!string.IsNullOrEmpty(fileListToDelete)) { updateFlags |= UpdateFlages.DELETE_FILES; using (var deleteListStream = new MemoryStream(Encoding.UTF8.GetBytes(fileListToDelete.ToString()))) { UploadFile(deleteListStream, $"{_deleteListFileName}"); } PrintTime($"Deleted file list '{_deleteListFileName}' uploaded"); if (!updateNeeded) { PrintTime($"Only delete old files"); return(updateFlags); } } else if (!updateNeeded) { PrintTime($"No update needed"); return(updateFlags); } updateFlags |= UpdateFlages.UPADTE_FILES; newCacheFileStream.Seek(0, SeekOrigin.Begin); UploadFile(newCacheFileStream, $"{_uploadCacheTempFileName}"); PrintTime($"New remote file cache '{_uploadCacheTempFileName}' uploaded"); } } } var tarGzStreamSize = tarGzStream.Length; tarGzStream.Seek(0, SeekOrigin.Begin); UploadFile(tarGzStream, _compressedUploadDiffContentFilename); PrintTime($"Compressed file diff '{_compressedUploadDiffContentFilename}' [{tarGzStreamSize,13:n0} bytes] uploaded"); } return(updateFlags); }