/// <summary>
/// Creates a chunker appropriate to the runtime environment
/// </summary>
public static IChunker Create(ChunkerConfiguration configuration)
{
// Enforcing check earlier:
// Use COMchunker only IFF avgchunksize = 64K, Windows 64bit and the module is present.
// See 'IsComChunkerSupported'.
if (configuration.AvgChunkSize == ChunkerConfiguration.SupportedComChunkerConfiguration.AvgChunkSize &&
IsComChunkerSupported &&
ComChunkerLoadError.Value == null)
{
try
{
return(new ComChunker(configuration));
}
catch (System.ComponentModel.Win32Exception)
{
// Some older versions of windows. Fall back to managed chunker.
}
catch (System.Threading.ThreadStateException)
{
// May happen in tests, when the thread apartment is not configured correctly. Fall back to managed chunker in this case as well.
}
}
return(new ManagedChunker(configuration));
}
static ChunkerConfiguration()
{
int avgChunkSize;
if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_AVG_CHUNK_SIZE"), out avgChunkSize))
{
avgChunkSize = 64 * 1024;
}
int minChunkSize;
if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MIN_CHUNK_SIZE"), out minChunkSize))
{
minChunkSize = avgChunkSize / 2;
}
int maxChunkSize;
if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MAX_CHUNK_SIZE"), out maxChunkSize))
{
maxChunkSize = avgChunkSize * 2;
}
Default = new ChunkerConfiguration(minChunkSize, avgChunkSize, maxChunkSize);
}
/// <nodoc />
public DedupNodeOrChunkHashAlgorithm(IChunker chunker)
{
if (!ChunkerConfiguration.IsValidChunkSize(chunker.Configuration))
{
throw new NotImplementedException($"Unsupported chunk size specified: {chunker.Configuration.AvgChunkSize} in bytes.");
}
_chunker = chunker;
Initialize();
}
/// <nodoc />
public static NodeAlgorithmId GetNodeAlgorithmId(ChunkerConfiguration chunkerConfiguration)
{
var hit = ChunkSizeToAlgorithmId.TryGetValue(chunkerConfiguration.AvgChunkSize, out var nodeAlgorithmId);
if (!hit)
{
throw new NotImplementedException($"{nameof(GetNodeAlgorithmId)}: No algorithm id found for chunker with avg chnk size: {chunkerConfiguration.AvgChunkSize} bytes.");
}
return(nodeAlgorithmId);
}
/// <summary>
/// Initializes a new instance of the <see cref="Chunker"/> class.
/// </summary>
public ComChunkerNonDeterministic(ChunkerConfiguration config)
{
if (Thread.CurrentThread.GetApartmentState() != ApartmentState.MTA)
{
throw new System.Threading.ThreadStateException("Thread must be in MTA ApartmentState");
}
Contract.Assert(
config.AvgChunkSize == ComChunker.SupportedAvgChunkSize,
"ComChunker only supports average chunk size of 64KB"
);
_chunkLibrary = NativeMethods.CreateChunkLibrary();
_chunkLibrary.InitializeForPushBuffers();
Action[] configures = new Action[]
{
() =>
{
var max = (object)(config.MaxChunkSize);
_chunkLibrary.SetParameter(NativeMethods.DEDUP_PT_MaxChunkSizeBytes, ref max);
},
() =>
{
// THIS IS NOT HONORED
var avg = (object)(config.AvgChunkSize);
_chunkLibrary.SetParameter(NativeMethods.DEDUP_PT_AvgChunkSizeBytes, ref avg);
},
() =>
{
var min = (object)(config.MinChunkSize);
_chunkLibrary.SetParameter(NativeMethods.DEDUP_PT_MinChunkSizeBytes, ref min);
},
};
// The order in which these need to be set depends on whether we are going bigger or smaller
// than the default.
if (config.MaxChunkSize < ChunkerConfiguration.SupportedComChunkerConfiguration.MaxChunkSize)
{
Array.Reverse(configures);
}
foreach (Action configure in configures)
{
configure();
}
object chunksEnum;
_chunkLibrary.StartChunking(IteratorComGuid, out chunksEnum);
_chunkHashIterator = (IDedupIterateChunksHash32)chunksEnum;
}
/// <summary>
/// Creates a chunker appropriate to the runtime environment
/// </summary>
public static IChunker Create(ChunkerConfiguration configuration)
{
if (IsComChunkerSupported && ComChunkerLoadError.Value == null)
{
try
{
return(new ComChunker(configuration));
}
catch (System.ComponentModel.Win32Exception)
{
// Some older versions of windows. Fall back to managed chunker.
}
}
return(new ManagedChunker(configuration));
}
static ChunkerConfiguration()
{
if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_AVG_CHUNK_SIZE"), out var avgChunkSize))
{
avgChunkSize = 64 * 1024;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MIN_CHUNK_SIZE"), out var minChunkSize))
{
minChunkSize = avgChunkSize / 2;
}
if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MAX_CHUNK_SIZE"), out var maxChunkSize))
{
maxChunkSize = avgChunkSize * 2;
}
SupportedComChunkerConfiguration = new ChunkerConfiguration(minChunkSize, avgChunkSize, maxChunkSize);
}
/// <summary>
/// Appends algorithm id to hash result.
/// </summary>
private byte[] SerializeHashAndId()
{
Contract.Assert(_lastNode != null);
var bytes = _lastNode.Value.Hash.ToArray();
byte[] result = new byte[bytes.Length + 1];
bytes.CopyTo(result, 0);
if (_lastNode.Value.Type == DedupNode.NodeType.ChunkLeaf)
{
result[bytes.Length] = ChunkDedupIdentifier.ChunkAlgorithmId;
}
else
{
result[bytes.Length] = (byte)ChunkerConfiguration.GetNodeAlgorithmId(_chunker.Configuration);
}
return(result);
}
/// <summary>
/// Creates a chunker appropriate to the runtime environment
/// </summary>
public static IChunker Create(ChunkerConfiguration configuration)
{
// Enforcing check earlier:
// Use COMchunker only IFF avgchunksize = 64K, Windows 64bit and the module is present.
// See 'IsComChunkerSupported'.
if (configuration.AvgChunkSize == ChunkerConfiguration.SupportedComChunkerConfiguration.AvgChunkSize &&
IsComChunkerSupported &&
ComChunkerLoadError.Value == null)
{
try
{
return(new ComChunker(configuration));
}
catch (System.ComponentModel.Win32Exception)
{
// Some older versions of windows. Fall back to managed chunker.
}
}
return(new ManagedChunker(configuration));
}
public ManagedChunkerNonDeterministic(ChunkerConfiguration configuration)
{
Configuration = configuration;
}
/// <summary>
///
/// </summary>
public ManagedChunker(ChunkerConfiguration configuration)
{
_inner = new DeterministicChunker(configuration, new ManagedChunkerNonDeterministic(configuration));
}
public DeterministicChunker(ChunkerConfiguration configuration, INonDeterministicChunker chunker)
{
Configuration = configuration;
_chunker = chunker;
_pushBufferPool = new ByteArrayPool(configuration.MinPushBufferSize);
}
/// <nodoc />
public static bool IsValidChunkSize(ChunkerConfiguration chunkerConfiguration)
{
return(
chunkerConfiguration.AvgChunkSize == HashType.Dedup64K.GetAvgChunkSize() ||
chunkerConfiguration.AvgChunkSize == HashType.Dedup1024K.GetAvgChunkSize());
}
/// <summary>
/// Initializes a new instance of the <see cref="DedupNodeHashAlgorithm"/> class.
/// </summary>
public DedupNodeHashAlgorithm(ChunkerConfiguration configuration, DedupNodeTree.Algorithm treeAlgorithm)
: this(treeAlgorithm, Chunker.Create(configuration))
{
}
