public static void CreateSingleChunk(TFChunkDbConfig config, int chunkNum, string filename, int?actualDataSize = null, bool isScavenged = false, byte[] contents = null) { var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkNum, chunkNum, isScavenged, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var dataSize = actualDataSize ?? config.ChunkSize; var buf = new byte[ChunkHeader.Size + dataSize + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); var chunkFooter = new ChunkFooter(true, true, dataSize, dataSize, 0, new byte[ChunkFooter.ChecksumSize]); chunkBytes = chunkFooter.AsByteArray(); Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length); if (contents != null) { if (contents.Length != dataSize) { throw new Exception("Wrong contents size."); } Buffer.BlockCopy(contents, 0, buf, ChunkHeader.Size, contents.Length); } File.WriteAllBytes(filename, buf); }
private ChunkFooter WriteFooter(ICollection <PosMap> mapping) { var workItem = _writerWorkItem; int mapSize = 0; if (mapping != null) { if (!_inMem && _isCached != 0) { throw new InvalidOperationException("Trying to write mapping while chunk is cached. " + "You probably are writing scavenged chunk as cached. " + "Do not do this."); } mapSize = mapping.Count * PosMap.FullSize; workItem.Buffer.SetLength(mapSize); workItem.Buffer.Position = 0; foreach (var map in mapping) { map.Write(workItem.BufferWriter); } if (_inMem) { ResizeMemStream(workItem, mapSize); } WriteRawData(workItem, workItem.Buffer); } workItem.FlushToDisk(); var bufferSize = workItem.StreamLength - workItem.StreamPosition - ChunkFooter.Size; Log.Debug("Buffer size is " + bufferSize); if (bufferSize > 0) { byte[] buffer = new byte[bufferSize]; WriteRawData(workItem, buffer, buffer.Length); } Flush(); var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]); //MD5 workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize); //FILE var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash); workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size); Flush(); // trying to prevent bug with resized file, but no data in it var fileSize = ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size; if (_chunkHeader.Version == (byte)ChunkVersions.Unaligned && workItem.StreamLength != fileSize) { //TODO GFY this is dead code as all chunks are now Aligned. Log.Debug("Resizing stream as header is unaligned"); workItem.ResizeStream(fileSize); _fileSize = fileSize; } return(footerWithHash); }
private ChunkFooter WriteFooter(ICollection <PosMap> mapping) { var workItem = _writerWorkItem; workItem.ResizeStream((int)workItem.StreamPosition); int mapSize = 0; if (mapping != null) { if (!_inMem && _isCached != 0) { throw new InvalidOperationException("Trying to write mapping while chunk is cached. " + "You probably are writing scavenged chunk as cached. " + "Do not do this."); } mapSize = mapping.Count * PosMap.FullSize; workItem.Buffer.SetLength(mapSize); workItem.Buffer.Position = 0; foreach (var map in mapping) { map.Write(workItem.BufferWriter); } if (_inMem) { ResizeMemStream(workItem, mapSize); } WriteRawData(workItem, workItem.Buffer); } workItem.FlushToDisk(); if (_chunkHeader.Version >= (byte)ChunkVersions.Aligned) { var alignedSize = GetAlignedSize(ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size); var bufferSize = alignedSize - workItem.StreamPosition - ChunkFooter.Size; Log.Debug("Buffer size is " + bufferSize); if (bufferSize > 0) { byte[] buffer = new byte[bufferSize]; WriteRawData(workItem, buffer, buffer.Length); } } Flush(); var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]); //MD5 workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize); //FILE var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash); workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size); Flush(); _fileSize = (int)workItem.StreamLength; return(footerWithHash); }
private void CreateChunk(string filename, int actualSize, int chunkSize) { var chunkHeader = new ChunkHeader(1, chunkSize, 0, 0, 0); var chunkBytes = chunkHeader.AsByteArray(); var buf = new byte[ChunkHeader.Size + actualSize + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); var chunkFooter = new ChunkFooter(true, actualSize, actualSize, 0, new byte[ChunkFooter.ChecksumSize]); chunkBytes = chunkFooter.AsByteArray(); Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length); File.WriteAllBytes(filename, buf); }
private ChunkFooter WriteFooter(ICollection <PosMap> mapping) { var workItem = _writerWorkItem; int mapSize = 0; if (mapping != null) { if (!_inMem && _isCached != 0) { throw new InvalidOperationException("Trying to write mapping while chunk is cached! " + "You probably are writing scavenged chunk as cached. " + "Don't do this!"); } mapSize = mapping.Count * PosMap.FullSize; workItem.Buffer.SetLength(mapSize); workItem.Buffer.Position = 0; foreach (var map in mapping) { map.Write(workItem.BufferWriter); } if (_inMem) { ResizeMemStream(workItem, mapSize); } WriteRawData(workItem, workItem.Buffer); } var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]); //MD5 workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize); //FILE var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash); workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size); Flush(); // trying to prevent bug with resized file, but no data in it var fileSize = ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size; if (workItem.StreamLength != fileSize) { workItem.ResizeStream(fileSize); _fileSize = fileSize; } return(footerWithHash); }
public static void CreateMultiChunk(TFChunkDbConfig config, int chunkStartNum, int chunkEndNum, string filename, int?physicalSize = null, long?logicalSize = null) { if (chunkStartNum > chunkEndNum) { throw new ArgumentException("chunkStartNum"); } var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkStartNum, chunkEndNum, true, Guid.NewGuid()); var chunkBytes = chunkHeader.AsByteArray(); var physicalDataSize = physicalSize ?? config.ChunkSize; var logicalDataSize = logicalSize ?? (chunkEndNum - chunkStartNum + 1) * config.ChunkSize; var buf = new byte[ChunkHeader.Size + physicalDataSize + ChunkFooter.Size]; Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length); var chunkFooter = new ChunkFooter(true, true, physicalDataSize, logicalDataSize, 0, new byte[ChunkFooter.ChecksumSize]); chunkBytes = chunkFooter.AsByteArray(); Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length); File.WriteAllBytes(filename, buf); }
private ChunkFooter WriteFooter(ICollection<PosMap> mapping) { var workItem = _writerWorkItem; int mapSize = 0; if (mapping != null) { if (!_inMem && _isCached != 0) { throw new InvalidOperationException("Trying to write mapping while chunk is cached! " + "You probably are writing scavenged chunk as cached. " + "Don't do this!"); } mapSize = mapping.Count * PosMap.FullSize; workItem.Buffer.SetLength(mapSize); workItem.Buffer.Position = 0; foreach (var map in mapping) { map.Write(workItem.BufferWriter); } if (_inMem) ResizeMemStream(workItem, mapSize); WriteRawData(workItem, workItem.Buffer); } var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]); //MD5 workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize); //FILE var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash); workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size); Flush(); // trying to prevent bug with resized file, but no data in it var fileSize = ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size; if (workItem.StreamLength != fileSize) { workItem.ResizeStream(fileSize); _fileSize = fileSize; } return footerWithHash; }
private ChunkFooter WriteFooter(ICollection<PosMap> mapping) { var workItem = _writerWorkItem; int mapSize = 0; if (mapping != null) { if (_isCached != 0) { throw new InvalidOperationException("Trying to write mapping while chunk is cached! " + "You probably are writing scavenged chunk as cached. " + "Don't do this!"); } mapSize = mapping.Count * sizeof(ulong); workItem.Buffer.SetLength(mapSize); workItem.Buffer.Position = 0; foreach (var map in mapping) { workItem.BufferWriter.Write(map.AsUInt64()); } WriteRawData(workItem, workItem.Buffer); } var footerNoHash = new ChunkFooter(true, _actualDataSize, _actualDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]); //MD5 workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize); //FILE var footerWithHash = new ChunkFooter(true, _actualDataSize, _actualDataSize, mapSize, workItem.MD5.Hash); workItem.Stream.Write(footerWithHash.AsByteArray(), 0, ChunkFooter.Size); var fileSize = ChunkHeader.Size + _actualDataSize + mapSize + ChunkFooter.Size; if (workItem.Stream.Length != fileSize) workItem.Stream.SetLength(fileSize); return footerWithHash; }