private RingNode[] CreateNewRing(ClusterState clusterState)
{
MachineId currentMachineId = clusterState.PrimaryMachineId;
SortedList <long, RingNode> sortedMachines = new SortedList <long, RingNode>();
IReadOnlyList <MachineLocation> machineLocations = clusterState.Locations;
MachineId machineId;
foreach (var machine in machineLocations)
{
// We use the machineId to place it in the virtual ring
bool success = clusterState.TryResolveMachineId(machine, out machineId);
// Create the consistent-hashing virtual ring with the active servers
if (success && !clusterState.IsMachineMarkedInactive(machineId) && currentMachineId != machineId)
{
long id = _contentHasher.GetContentHash(
BitConverter.GetBytes(machineId.Index)
).LeastSignificantLong();
sortedMachines.Add(id, new RingNode(id, machine));
}
}
return(sortedMachines.Values.ToArray());
}
private void Run(string method, int contentSize, int iterations = Iterations)
{
var content = ThreadSafeRandom.GetBytes(contentSize);
PerfTestCase(); // warm up
var stopwatch = Stopwatch.StartNew();
for (var i = 0; i < iterations; i++)
{
PerfTestCase();
}
stopwatch.Stop();
var rate = (long)(iterations / stopwatch.Elapsed.TotalSeconds);
var name = GetType().Name + "." + method;
_resultsFixture.AddResults(Output, name, rate, "items/sec", iterations);
void PerfTestCase()
{
var x = _hasher.GetContentHash(content);
if (x.ByteLength != _hasher.Info.ByteLength)
{
throw new InvalidOperationException("unexpected hash length");
}
}
}
internal static async Task <bool> VerifyStreamAsync(Stream stream, IList <ChunkInfo> expectedChunks, ChunkDedupedFileContentHash expectedHash, CancellationToken cancellationToken)
{
ulong totalBytesChunked = 0;
var producedChunks = new List <ChunkInfo>(expectedChunks.Count);
var maxChunkSize = expectedChunks.Max((chunk) => chunk.Size);
var buffer = new byte[maxChunkSize];
foreach (var currentChunk in expectedChunks)
{
int bytesRead = await stream.ReadAsync(buffer, 0, (int)currentChunk.Size, cancellationToken);
if (bytesRead != currentChunk.Size)
{
return(false);
}
byte[] chunkHash = ChunkHasher.GetContentHash(
buffer,
0,
bytesRead).ToHashByteArray();
if (!chunkHash.SequenceEqual(currentChunk.Hash))
{
// Hash mismatch
return(false);
}
producedChunks.Add(new ChunkInfo(
totalBytesChunked,
currentChunk.Size,
chunkHash));
totalBytesChunked += (ulong)bytesRead;
}
if (stream.ReadByte() != -1)
{
// File content is longer
return(false);
}
var node = DedupNode.Create(producedChunks);
var hashBytesExcludingAlgorithm = node.Hash.Take(DedupSingleChunkHashInfo.Length).ToArray();
var actualHash = new ChunkDedupedFileContentHash(hashBytesExcludingAlgorithm);
return(expectedHash == actualHash);
}
private (byte[] bytes, DedupNode contentNode) GetTestContent(int numberOfChunks, int chunkSize)
{
byte[] buffer = new byte[chunkSize * numberOfChunks];
List <ChunkInfo> storedChunks = new List <ChunkInfo>();
var random = new Random();
random.NextBytes(buffer);
ulong offset = 0;
for (int i = 0; i < numberOfChunks; i++)
{
var newChunk = new ChunkInfo(
offset,
(uint)chunkSize,
ChunkHasher.GetContentHash(buffer, (int)offset, chunkSize).ToHashByteArray());
storedChunks.Add(newChunk);
offset += (ulong)chunkSize;
}
return(buffer, DedupNode.Create(storedChunks));
}
/// <summary>
/// Computes a hash for a current machine location.
/// </summary>
public ContentHash GetContentHash(IContentHasher hasher)
{
Contract.Requires(IsValid, "Please do not use default struct instance");
return(hasher.GetContentHash(Data));
}
/// <nodoc />
public static NodeDedupIdentifier CalculateNodeDedupIdentifier(this DedupNode node, HashType hashType)
{
return(new NodeDedupIdentifier(ChunkHasher.GetContentHash(node.Serialize()).ToHashByteArray(), hashType.GetNodeAlgorithmId()));
}
/// <nodoc />
public static NodeDedupIdentifier CalculateIdentifierFromSerializedNode(byte[] bytes, HashType hashType)
{
Contract.Requires(bytes != null);
Contract.Check(((NodeAlgorithmId)hashType.GetNodeAlgorithmId()).IsValidNode())?.Assert($"Cannot serialize from hash because hash type is invalid: {hashType}");
return(new NodeDedupIdentifier(Hasher.GetContentHash(bytes).ToHashByteArray(), hashType.GetNodeAlgorithmId()));
}
public static ChunkDedupIdentifier CalculateIdentifier(byte[] bytes)
{
Contract.Requires(bytes != null);
return(new ChunkDedupIdentifier(chunkHasher.GetContentHash(bytes).ToHashByteArray()));
}
