/// <summary> /// Creates a chunker appropriate to the runtime environment /// </summary> public static IChunker Create(ChunkerConfiguration configuration) { // Enforcing check earlier: // Use COMchunker only IFF avgchunksize = 64K, Windows 64bit and the module is present. // See 'IsComChunkerSupported'. if (configuration.AvgChunkSize == ChunkerConfiguration.SupportedComChunkerConfiguration.AvgChunkSize && IsComChunkerSupported && ComChunkerLoadError.Value == null) { try { return(new ComChunker(configuration)); } catch (System.ComponentModel.Win32Exception) { // Some older versions of windows. Fall back to managed chunker. } catch (System.Threading.ThreadStateException) { // May happen in tests, when the thread apartment is not configured correctly. Fall back to managed chunker in this case as well. } } return(new ManagedChunker(configuration)); }
static ChunkerConfiguration() { int avgChunkSize; if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_AVG_CHUNK_SIZE"), out avgChunkSize)) { avgChunkSize = 64 * 1024; } int minChunkSize; if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MIN_CHUNK_SIZE"), out minChunkSize)) { minChunkSize = avgChunkSize / 2; } int maxChunkSize; if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MAX_CHUNK_SIZE"), out maxChunkSize)) { maxChunkSize = avgChunkSize * 2; } Default = new ChunkerConfiguration(minChunkSize, avgChunkSize, maxChunkSize); }
/// <nodoc /> public DedupNodeOrChunkHashAlgorithm(IChunker chunker) { if (!ChunkerConfiguration.IsValidChunkSize(chunker.Configuration)) { throw new NotImplementedException($"Unsupported chunk size specified: {chunker.Configuration.AvgChunkSize} in bytes."); } _chunker = chunker; Initialize(); }
/// <nodoc /> public static NodeAlgorithmId GetNodeAlgorithmId(ChunkerConfiguration chunkerConfiguration) { var hit = ChunkSizeToAlgorithmId.TryGetValue(chunkerConfiguration.AvgChunkSize, out var nodeAlgorithmId); if (!hit) { throw new NotImplementedException($"{nameof(GetNodeAlgorithmId)}: No algorithm id found for chunker with avg chnk size: {chunkerConfiguration.AvgChunkSize} bytes."); } return(nodeAlgorithmId); }
/// <summary> /// Initializes a new instance of the <see cref="Chunker"/> class. /// </summary> public ComChunkerNonDeterministic(ChunkerConfiguration config) { if (Thread.CurrentThread.GetApartmentState() != ApartmentState.MTA) { throw new System.Threading.ThreadStateException("Thread must be in MTA ApartmentState"); } Contract.Assert( config.AvgChunkSize == ComChunker.SupportedAvgChunkSize, "ComChunker only supports average chunk size of 64KB" ); _chunkLibrary = NativeMethods.CreateChunkLibrary(); _chunkLibrary.InitializeForPushBuffers(); Action[] configures = new Action[] { () => { var max = (object)(config.MaxChunkSize); _chunkLibrary.SetParameter(NativeMethods.DEDUP_PT_MaxChunkSizeBytes, ref max); }, () => { // THIS IS NOT HONORED var avg = (object)(config.AvgChunkSize); _chunkLibrary.SetParameter(NativeMethods.DEDUP_PT_AvgChunkSizeBytes, ref avg); }, () => { var min = (object)(config.MinChunkSize); _chunkLibrary.SetParameter(NativeMethods.DEDUP_PT_MinChunkSizeBytes, ref min); }, }; // The order in which these need to be set depends on whether we are going bigger or smaller // than the default. if (config.MaxChunkSize < ChunkerConfiguration.SupportedComChunkerConfiguration.MaxChunkSize) { Array.Reverse(configures); } foreach (Action configure in configures) { configure(); } object chunksEnum; _chunkLibrary.StartChunking(IteratorComGuid, out chunksEnum); _chunkHashIterator = (IDedupIterateChunksHash32)chunksEnum; }
/// <summary> /// Creates a chunker appropriate to the runtime environment /// </summary> public static IChunker Create(ChunkerConfiguration configuration) { if (IsComChunkerSupported && ComChunkerLoadError.Value == null) { try { return(new ComChunker(configuration)); } catch (System.ComponentModel.Win32Exception) { // Some older versions of windows. Fall back to managed chunker. } } return(new ManagedChunker(configuration)); }
static ChunkerConfiguration() { if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_AVG_CHUNK_SIZE"), out var avgChunkSize)) { avgChunkSize = 64 * 1024; } if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MIN_CHUNK_SIZE"), out var minChunkSize)) { minChunkSize = avgChunkSize / 2; } if (!int.TryParse(Environment.GetEnvironmentVariable("BUILDXL_TEST_MAX_CHUNK_SIZE"), out var maxChunkSize)) { maxChunkSize = avgChunkSize * 2; } SupportedComChunkerConfiguration = new ChunkerConfiguration(minChunkSize, avgChunkSize, maxChunkSize); }
/// <summary> /// Appends algorithm id to hash result. /// </summary> private byte[] SerializeHashAndId() { Contract.Assert(_lastNode != null); var bytes = _lastNode.Value.Hash.ToArray(); byte[] result = new byte[bytes.Length + 1]; bytes.CopyTo(result, 0); if (_lastNode.Value.Type == DedupNode.NodeType.ChunkLeaf) { result[bytes.Length] = ChunkDedupIdentifier.ChunkAlgorithmId; } else { result[bytes.Length] = (byte)ChunkerConfiguration.GetNodeAlgorithmId(_chunker.Configuration); } return(result); }
/// <summary> /// Creates a chunker appropriate to the runtime environment /// </summary> public static IChunker Create(ChunkerConfiguration configuration) { // Enforcing check earlier: // Use COMchunker only IFF avgchunksize = 64K, Windows 64bit and the module is present. // See 'IsComChunkerSupported'. if (configuration.AvgChunkSize == ChunkerConfiguration.SupportedComChunkerConfiguration.AvgChunkSize && IsComChunkerSupported && ComChunkerLoadError.Value == null) { try { return(new ComChunker(configuration)); } catch (System.ComponentModel.Win32Exception) { // Some older versions of windows. Fall back to managed chunker. } } return(new ManagedChunker(configuration)); }
public ManagedChunkerNonDeterministic(ChunkerConfiguration configuration) { Configuration = configuration; }
/// <summary> /// /// </summary> public ManagedChunker(ChunkerConfiguration configuration) { _inner = new DeterministicChunker(configuration, new ManagedChunkerNonDeterministic(configuration)); }
public DeterministicChunker(ChunkerConfiguration configuration, INonDeterministicChunker chunker) { Configuration = configuration; _chunker = chunker; _pushBufferPool = new ByteArrayPool(configuration.MinPushBufferSize); }
/// <nodoc /> public static bool IsValidChunkSize(ChunkerConfiguration chunkerConfiguration) { return( chunkerConfiguration.AvgChunkSize == HashType.Dedup64K.GetAvgChunkSize() || chunkerConfiguration.AvgChunkSize == HashType.Dedup1024K.GetAvgChunkSize()); }
/// <summary> /// Initializes a new instance of the <see cref="DedupNodeHashAlgorithm"/> class. /// </summary> public DedupNodeHashAlgorithm(ChunkerConfiguration configuration, DedupNodeTree.Algorithm treeAlgorithm) : this(treeAlgorithm, Chunker.Create(configuration)) { }