private RingNode[] CreateNewRing(ClusterState clusterState) { MachineId currentMachineId = clusterState.PrimaryMachineId; SortedList <long, RingNode> sortedMachines = new SortedList <long, RingNode>(); IReadOnlyList <MachineLocation> machineLocations = clusterState.Locations; MachineId machineId; foreach (var machine in machineLocations) { // We use the machineId to place it in the virtual ring bool success = clusterState.TryResolveMachineId(machine, out machineId); // Create the consistent-hashing virtual ring with the active servers if (success && !clusterState.IsMachineMarkedInactive(machineId) && currentMachineId != machineId) { long id = _contentHasher.GetContentHash( BitConverter.GetBytes(machineId.Index) ).LeastSignificantLong(); sortedMachines.Add(id, new RingNode(id, machine)); } } return(sortedMachines.Values.ToArray()); }
private void Run(string method, int contentSize, int iterations = Iterations) { var content = ThreadSafeRandom.GetBytes(contentSize); PerfTestCase(); // warm up var stopwatch = Stopwatch.StartNew(); for (var i = 0; i < iterations; i++) { PerfTestCase(); } stopwatch.Stop(); var rate = (long)(iterations / stopwatch.Elapsed.TotalSeconds); var name = GetType().Name + "." + method; _resultsFixture.AddResults(Output, name, rate, "items/sec", iterations); void PerfTestCase() { var x = _hasher.GetContentHash(content); if (x.ByteLength != _hasher.Info.ByteLength) { throw new InvalidOperationException("unexpected hash length"); } } }
internal static async Task <bool> VerifyStreamAsync(Stream stream, IList <ChunkInfo> expectedChunks, ChunkDedupedFileContentHash expectedHash, CancellationToken cancellationToken) { ulong totalBytesChunked = 0; var producedChunks = new List <ChunkInfo>(expectedChunks.Count); var maxChunkSize = expectedChunks.Max((chunk) => chunk.Size); var buffer = new byte[maxChunkSize]; foreach (var currentChunk in expectedChunks) { int bytesRead = await stream.ReadAsync(buffer, 0, (int)currentChunk.Size, cancellationToken); if (bytesRead != currentChunk.Size) { return(false); } byte[] chunkHash = ChunkHasher.GetContentHash( buffer, 0, bytesRead).ToHashByteArray(); if (!chunkHash.SequenceEqual(currentChunk.Hash)) { // Hash mismatch return(false); } producedChunks.Add(new ChunkInfo( totalBytesChunked, currentChunk.Size, chunkHash)); totalBytesChunked += (ulong)bytesRead; } if (stream.ReadByte() != -1) { // File content is longer return(false); } var node = DedupNode.Create(producedChunks); var hashBytesExcludingAlgorithm = node.Hash.Take(DedupSingleChunkHashInfo.Length).ToArray(); var actualHash = new ChunkDedupedFileContentHash(hashBytesExcludingAlgorithm); return(expectedHash == actualHash); }
private (byte[] bytes, DedupNode contentNode) GetTestContent(int numberOfChunks, int chunkSize) { byte[] buffer = new byte[chunkSize * numberOfChunks]; List <ChunkInfo> storedChunks = new List <ChunkInfo>(); var random = new Random(); random.NextBytes(buffer); ulong offset = 0; for (int i = 0; i < numberOfChunks; i++) { var newChunk = new ChunkInfo( offset, (uint)chunkSize, ChunkHasher.GetContentHash(buffer, (int)offset, chunkSize).ToHashByteArray()); storedChunks.Add(newChunk); offset += (ulong)chunkSize; } return(buffer, DedupNode.Create(storedChunks)); }
/// <summary> /// Computes a hash for a current machine location. /// </summary> public ContentHash GetContentHash(IContentHasher hasher) { Contract.Requires(IsValid, "Please do not use default struct instance"); return(hasher.GetContentHash(Data)); }
/// <nodoc /> public static NodeDedupIdentifier CalculateNodeDedupIdentifier(this DedupNode node, HashType hashType) { return(new NodeDedupIdentifier(ChunkHasher.GetContentHash(node.Serialize()).ToHashByteArray(), hashType.GetNodeAlgorithmId())); }
/// <nodoc /> public static NodeDedupIdentifier CalculateIdentifierFromSerializedNode(byte[] bytes, HashType hashType) { Contract.Requires(bytes != null); Contract.Check(((NodeAlgorithmId)hashType.GetNodeAlgorithmId()).IsValidNode())?.Assert($"Cannot serialize from hash because hash type is invalid: {hashType}"); return(new NodeDedupIdentifier(Hasher.GetContentHash(bytes).ToHashByteArray(), hashType.GetNodeAlgorithmId())); }
public static ChunkDedupIdentifier CalculateIdentifier(byte[] bytes) { Contract.Requires(bytes != null); return(new ChunkDedupIdentifier(chunkHasher.GetContentHash(bytes).ToHashByteArray())); }