public static UpdateCacheEntry ReadFromStream(BinaryReader reader)
{
var fileEntry = new UpdateCacheEntry
{
FilePath = reader.ReadString(),
LastWriteTimeUtcTicks = reader.ReadInt64(),
FileSize = reader.ReadInt64(),
};
return(fileEntry);
}
private void DownloadAndCalculateChangedFiles(ConcurrentDictionary <string, FileInfo> localFileCache, StringBuilder fileListToDelete, ConcurrentDictionary <string, bool> filesNoUpdateNeeded)
{
try
{
using (MemoryStream currentRemoteCacheFile = new MemoryStream())
{
DownloadFile(_uploadCacheFileName, currentRemoteCacheFile);
PrintTime($"Remote file cache '{_uploadCacheFileName}' downloaded");
currentRemoteCacheFile.Seek(0, SeekOrigin.Begin);
using (BinaryReader reader = new BinaryReader(currentRemoteCacheFile))
{
int entryCount = reader.ReadInt32();
int deleteFileCount = 0;
long deleteFileSize = 0;
int upToDateFileCount = 0;
long upToDateFileSize = 0;
long remoteFilesSize = 0;
for (int i = 0; i < entryCount; i++)
{
var remotefileEntry = UpdateCacheEntry.ReadFromStream(reader);
remoteFilesSize += remotefileEntry.FileSize;
FileInfo localFileInfo;
if (!localFileCache.TryGetValue(remotefileEntry.FilePath, out localFileInfo))
{
deleteFileCount++;
deleteFileSize += remotefileEntry.FileSize;
fileListToDelete.Append(remotefileEntry.FilePath).Append("\n");
}
else if (localFileInfo.LastWriteTimeUtc.Ticks == remotefileEntry.LastWriteTimeUtcTicks && localFileInfo.Length == remotefileEntry.FileSize)
{
upToDateFileCount++;
upToDateFileSize += remotefileEntry.FileSize;
filesNoUpdateNeeded[remotefileEntry.FilePath] = true;
}
}
PrintTime($"{deleteFileCount,7:n0} [{deleteFileSize,13:n0} bytes] of {entryCount,7:n0} [{remoteFilesSize,13:n0} bytes] files need to be deleted");
PrintTime($"{upToDateFileCount,7:n0} [{upToDateFileSize,13:n0} bytes] of {entryCount,7:n0} [{remoteFilesSize,13:n0} bytes] files don't need to be updated");
}
}
}
catch (Exception ex)
{
// TODO log verbose messages differently
var msg = ex.Message;
PrintTime($"Remote file cache '{_uploadCacheFileName}' not found! We are uploading all files!");
}
PrintTime($"Diff between local and remote file cache calculated");
}
private UpdateFlages CreateAndUploadFileDiff(ConcurrentDictionary <string, FileInfo> localFileCache, ConcurrentDictionary <string, bool> filesNoUpdateNeeded, string fileListToDelete)
{
var updateFlags = UpdateFlages.NONE;
using (Stream tarGzStream = new MemoryStream())
{
using (var tarGzWriter = WriterFactory.Open(tarGzStream, ArchiveType.Tar, CompressionType.GZip))
{
using (MemoryStream newCacheFileStream = new MemoryStream())
{
using (BinaryWriter newCacheFileWriter = new BinaryWriter(newCacheFileStream))
{
newCacheFileWriter.Write(localFileCache.Count);
var updateNeeded = false;
var updateFileCount = 0;
long updateFileSize = 0;
var allFileCount = 0;
long allFileSize = 0;
foreach (var file in localFileCache)
{
allFileCount++;
allFileSize += file.Value.Length;
// add new cache file entry
UpdateCacheEntry.WriteToStream(newCacheFileWriter, file.Key, file.Value);
// add new file entry
if (filesNoUpdateNeeded.ContainsKey(file.Key))
{
continue;
}
updateNeeded = true;
updateFileCount++;
updateFileSize += file.Value.Length;
try
{
tarGzWriter.Write(file.Key, file.Value);
}
catch (IOException ioEx)
{
PrintError(ioEx, withStacktrace: false);
}
catch (Exception ex)
{
PrintError(ex);
}
}
PrintTime($"{updateFileCount,7:n0} [{updateFileSize,13:n0} bytes] of {allFileCount,7:n0} [{allFileSize,13:n0} bytes] files need to be updated");
if (!string.IsNullOrEmpty(fileListToDelete))
{
updateFlags |= UpdateFlages.DELETE_FILES;
using (var deleteListStream = new MemoryStream(Encoding.UTF8.GetBytes(fileListToDelete.ToString())))
{
UploadFile(deleteListStream, $"{_deleteListFileName}");
}
PrintTime($"Deleted file list '{_deleteListFileName}' uploaded");
if (!updateNeeded)
{
PrintTime($"Only delete old files");
return(updateFlags);
}
}
else if (!updateNeeded)
{
PrintTime($"No update needed");
return(updateFlags);
}
updateFlags |= UpdateFlages.UPADTE_FILES;
newCacheFileStream.Seek(0, SeekOrigin.Begin);
UploadFile(newCacheFileStream, $"{_uploadCacheTempFileName}");
PrintTime($"New remote file cache '{_uploadCacheTempFileName}' uploaded");
}
}
}
var tarGzStreamSize = tarGzStream.Length;
tarGzStream.Seek(0, SeekOrigin.Begin);
UploadFile(tarGzStream, _compressedUploadDiffContentFilename);
PrintTime($"Compressed file diff '{_compressedUploadDiffContentFilename}' [{tarGzStreamSize,13:n0} bytes] uploaded");
}
return(updateFlags);
}
