public void Int32Roundtrip() { var buffer = new byte[4]; var context1 = new BufferSerializeContext(buffer); context1.Serialize(7); var context2 = new BufferSerializeContext(buffer); var i = context2.DeserializeInt32(); i.Should().Be(7); }
public void GuidRoundtrip() { var buffer = new byte[16]; var id1 = new Guid("0647396E-C99F-47DB-B567-C022E953CB69"); var context1 = new BufferSerializeContext(buffer); context1.Serialize(id1); var context2 = new BufferSerializeContext(buffer); var id2 = context2.DeserializeGuid(); id2.Should().Be(id1); }
public void Int64Roundtrip() { var buffer = new byte[16]; var ticks = DateTime.UtcNow.Ticks; var context1 = new BufferSerializeContext(buffer); context1.Serialize(ticks); context1.Serialize(ticks + 1); var context2 = new BufferSerializeContext(buffer); var t1 = context2.DeserializeInt64(); var t2 = context2.DeserializeInt64(); t1.Should().Be(ticks); t2.Should().Be(ticks + 1); }
private async Task <MemoryContentDirectoryHeader> DeserializeHeaderAsync(Context context, AbsolutePath path) { try { var directoryHeader = new MemoryContentDirectoryHeader(); using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read)) { var header = new byte[BinaryHeaderSizeV2]; stream.Read(header, 0, header.Length); var headerContext = new BufferSerializeContext(header); directoryHeader.MagicFlag = headerContext.DeserializeByte(); if (directoryHeader.MagicFlag != BinaryFormatMagicFlag) { throw new CacheException("{0} binary format missing magic flag", Name); } directoryHeader.Version = headerContext.DeserializeByte(); directoryHeader.EntryCount = headerContext.DeserializeInt32(); if (directoryHeader.Version == BinaryFormatVersion) { header = new byte[BinaryHeaderExtraSizeV3]; stream.Read(header, 0, header.Length); headerContext = new BufferSerializeContext(header); directoryHeader.ContentSize = headerContext.DeserializeInt64(); directoryHeader.ReplicaCount = headerContext.DeserializeInt64(); directoryHeader.HeaderSize = BinaryHeaderSizeV2 + BinaryHeaderExtraSizeV3; } else { throw new CacheException("{0} expected {1} but read binary format version {2}", Name, BinaryFormatVersion, directoryHeader.Version); } return(directoryHeader); } } catch (Exception exception) { context.Warning($"{Name} failed to deserialize header of {FilePath} - starting with empty directory: {exception}"); return(new MemoryContentDirectoryHeader()); } }
public void OffsetPropertyInitialized() { var context = new BufferSerializeContext(new byte[0]); context.Offset.Should().Be(0); }
public void LengthProperty() { var context = new BufferSerializeContext(new byte[0]); context.Length.Should().Be(0); }
private async Task <ContentMap> DeserializeBodyAsync(Context context, MemoryContentDirectoryHeader header, AbsolutePath path, bool isLoadingBackup) { var contentDirectory = new ContentMap(); try { var sw = Stopwatch.StartNew(); using (var stream = await _fileSystem.OpenSafeAsync(path, FileAccess.Read, FileMode.Open, FileShare.Read)) { byte[] headerBuffer = new byte[header.HeaderSize]; stream.Read(headerBuffer, 0, header.HeaderSize); var streamSync = new object(); var entriesSync = new object(); var entries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(header.EntryCount); var entriesRemaining = header.EntryCount; var tasks = new List <Task>(); var nowFileTimeUtc = DateTime.UtcNow.ToFileTimeUtc(); long totalSize = 0; long totalUniqueSize = 0; long oldestContentAccessTimeUtc = nowFileTimeUtc; long totalReplicaCount = 0; var statsLock = new object(); Action <int> readChunk = count => { var bufferLength = count * BinaryEntrySize; var buffer = new byte[bufferLength]; int bytesRead; lock (streamSync) { bytesRead = stream.Read(buffer, 0, bufferLength); } if (bytesRead != buffer.Length) { return; } var serializeContext = new BufferSerializeContext(buffer); var partitionEntries = new List <KeyValuePair <ContentHash, ContentFileInfo> >(count); for (var i = 0; i < count; i++) { var contentHash = serializeContext.DeserializeFullContentHash(); var fileSize = serializeContext.DeserializeInt64(); var lastAccessedFileTimeUtc = serializeContext.DeserializeInt64(); // Guard against corruption of serialized timestamps which affect LRU. If we get something out of range, // force it to now. if (lastAccessedFileTimeUtc < 0 || lastAccessedFileTimeUtc > nowFileTimeUtc) { lastAccessedFileTimeUtc = nowFileTimeUtc; } // ReSharper disable once UnusedVariable var accessCount = serializeContext.DeserializeInt32(); var replicaCount = serializeContext.DeserializeInt32(); var contentFileInfo = new ContentFileInfo(fileSize, lastAccessedFileTimeUtc, replicaCount); Interlocked.Add(ref totalSize, fileSize * replicaCount); Interlocked.Add(ref totalUniqueSize, fileSize); Interlocked.Add(ref totalReplicaCount, replicaCount); if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc) { lock (statsLock) { if (oldestContentAccessTimeUtc > lastAccessedFileTimeUtc) { oldestContentAccessTimeUtc = lastAccessedFileTimeUtc; } } } partitionEntries.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, contentFileInfo)); } lock (entriesSync) { entries.AddRange(partitionEntries); } }; while (entriesRemaining > 0) { var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk); tasks.Add(Task.Run(() => readChunk(chunkCount))); entriesRemaining -= chunkCount; } await TaskSafetyHelpers.WhenAll(tasks); context.Debug($"{Name}: Loaded content directory with {entries.Count} entries by {sw.ElapsedMilliseconds}ms: TotalContentSize={totalSize}, TotalUniqueSize={totalUniqueSize}, TotalReplicaCount={totalReplicaCount}, OldestContentTime={DateTime.FromFileTimeUtc(oldestContentAccessTimeUtc)}."); if (entries.Count == header.EntryCount) { contentDirectory = new ContentMap(entries); } else { throw new CacheException($"Failed to read expected number of entries. Entries.Count={entries.Count}, Header.EntryCount={header.EntryCount}."); } } if (!isLoadingBackup) { // At this point, we've either successfully read the file or tried and failed. Either way, the existing file should now be // deleted. On a clean shutdown, it will be regenerated. On a dirty shutdown, we want it to already be gone otherwise // we'll read in a file that is out-of-date. _fileSystem.MoveFile(path, _backupFilePath, true); } } catch (Exception exception) { context.Warning($"{Name} failed to deserialize {FilePath} - starting with empty directory: {exception}"); contentDirectory.Clear(); } return(contentDirectory); }
private async Task SerializeAsync() { if (ContentDirectory == null || ContentDirectory.Count == 0) { return; } var openTask = _fileSystem.OpenSafeAsync(FilePath, FileAccess.Write, FileMode.Create, FileShare.Delete); var sync = new object(); var writeHeader = true; var entries = ContentDirectory.ToArray(); var entriesRemaining = entries.Length; var startIndex = 0; var tasks = new List <Task>(); GetSizeAndReplicaCount(out var contentSize, out var replicaCount); using (var stream = await openTask) { Action <int, int> writeChunk = (index, count) => { var endIndexExclusive = index + count; var entryCount = endIndexExclusive - index; var bufferLength = entryCount * BinaryEntrySize; var partitionBuffer = new byte[bufferLength]; var partitionContext = new BufferSerializeContext(partitionBuffer); for (var i = index; i < endIndexExclusive; i++) { ContentFileInfo entry = entries[i].Value; partitionContext.SerializeFull(entries[i].Key); partitionContext.Serialize(entry.FileSize); partitionContext.Serialize(entry.LastAccessedFileTimeUtc); partitionContext.Serialize(UnusedAccessCount); partitionContext.Serialize(entry.ReplicaCount); } lock (sync) { if (writeHeader) { writeHeader = false; var headerBuffer = new byte[22]; var headerContext = new BufferSerializeContext(headerBuffer); headerContext.Serialize(BinaryFormatMagicFlag); headerContext.Serialize(BinaryFormatVersion); headerContext.Serialize(entries.Length); headerContext.Serialize(contentSize); headerContext.Serialize(replicaCount); stream.Write(headerBuffer, 0, headerBuffer.Length); } stream.Write(partitionBuffer, 0, partitionContext.Offset); } }; while (startIndex < entries.Length) { var i = startIndex; var chunkCount = Math.Min(entriesRemaining, BinaryMaxEntriesPerChunk); tasks.Add(Task.Run(() => writeChunk(i, chunkCount))); startIndex += chunkCount; entriesRemaining -= chunkCount; } await TaskSafetyHelpers.WhenAll(tasks); Contract.Assert(startIndex == entries.Length); } }