public static void CreateSingleChunk(TFChunkDbConfig config, int chunkNum, string filename,
int?actualDataSize = null, bool isScavenged = false, byte[] contents = null)
{
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkNum, chunkNum, isScavenged, Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var dataSize = actualDataSize ?? config.ChunkSize;
var buf = new byte[ChunkHeader.Size + dataSize + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
var chunkFooter = new ChunkFooter(true, true, dataSize, dataSize, 0, new byte[ChunkFooter.ChecksumSize]);
chunkBytes = chunkFooter.AsByteArray();
Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length);
if (contents != null)
{
if (contents.Length != dataSize)
{
throw new Exception("Wrong contents size.");
}
Buffer.BlockCopy(contents, 0, buf, ChunkHeader.Size, contents.Length);
}
File.WriteAllBytes(filename, buf);
}
private ChunkFooter WriteFooter(ICollection <PosMap> mapping)
{
var workItem = _writerWorkItem;
int mapSize = 0;
if (mapping != null)
{
if (!_inMem && _isCached != 0)
{
throw new InvalidOperationException("Trying to write mapping while chunk is cached. "
+ "You probably are writing scavenged chunk as cached. "
+ "Do not do this.");
}
mapSize = mapping.Count * PosMap.FullSize;
workItem.Buffer.SetLength(mapSize);
workItem.Buffer.Position = 0;
foreach (var map in mapping)
{
map.Write(workItem.BufferWriter);
}
if (_inMem)
{
ResizeMemStream(workItem, mapSize);
}
WriteRawData(workItem, workItem.Buffer);
}
workItem.FlushToDisk();
var bufferSize = workItem.StreamLength - workItem.StreamPosition - ChunkFooter.Size;
Log.Debug("Buffer size is " + bufferSize);
if (bufferSize > 0)
{
byte[] buffer = new byte[bufferSize];
WriteRawData(workItem, buffer, buffer.Length);
}
Flush();
var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]);
//MD5
workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize);
//FILE
var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash);
workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size);
Flush(); // trying to prevent bug with resized file, but no data in it
var fileSize = ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size;
if (_chunkHeader.Version == (byte)ChunkVersions.Unaligned && workItem.StreamLength != fileSize)
{
//TODO GFY this is dead code as all chunks are now Aligned.
Log.Debug("Resizing stream as header is unaligned");
workItem.ResizeStream(fileSize);
_fileSize = fileSize;
}
return(footerWithHash);
}
private ChunkFooter WriteFooter(ICollection <PosMap> mapping)
{
var workItem = _writerWorkItem;
workItem.ResizeStream((int)workItem.StreamPosition);
int mapSize = 0;
if (mapping != null)
{
if (!_inMem && _isCached != 0)
{
throw new InvalidOperationException("Trying to write mapping while chunk is cached. "
+ "You probably are writing scavenged chunk as cached. "
+ "Do not do this.");
}
mapSize = mapping.Count * PosMap.FullSize;
workItem.Buffer.SetLength(mapSize);
workItem.Buffer.Position = 0;
foreach (var map in mapping)
{
map.Write(workItem.BufferWriter);
}
if (_inMem)
{
ResizeMemStream(workItem, mapSize);
}
WriteRawData(workItem, workItem.Buffer);
}
workItem.FlushToDisk();
if (_chunkHeader.Version >= (byte)ChunkVersions.Aligned)
{
var alignedSize = GetAlignedSize(ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size);
var bufferSize = alignedSize - workItem.StreamPosition - ChunkFooter.Size;
Log.Debug("Buffer size is " + bufferSize);
if (bufferSize > 0)
{
byte[] buffer = new byte[bufferSize];
WriteRawData(workItem, buffer, buffer.Length);
}
}
Flush();
var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]);
//MD5
workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize);
//FILE
var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash);
workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size);
Flush();
_fileSize = (int)workItem.StreamLength;
return(footerWithHash);
}
private void CreateChunk(string filename, int actualSize, int chunkSize)
{
var chunkHeader = new ChunkHeader(1, chunkSize, 0, 0, 0);
var chunkBytes = chunkHeader.AsByteArray();
var buf = new byte[ChunkHeader.Size + actualSize + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
var chunkFooter = new ChunkFooter(true, actualSize, actualSize, 0, new byte[ChunkFooter.ChecksumSize]);
chunkBytes = chunkFooter.AsByteArray();
Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
}
private ChunkFooter WriteFooter(ICollection <PosMap> mapping)
{
var workItem = _writerWorkItem;
int mapSize = 0;
if (mapping != null)
{
if (!_inMem && _isCached != 0)
{
throw new InvalidOperationException("Trying to write mapping while chunk is cached! "
+ "You probably are writing scavenged chunk as cached. "
+ "Don't do this!");
}
mapSize = mapping.Count * PosMap.FullSize;
workItem.Buffer.SetLength(mapSize);
workItem.Buffer.Position = 0;
foreach (var map in mapping)
{
map.Write(workItem.BufferWriter);
}
if (_inMem)
{
ResizeMemStream(workItem, mapSize);
}
WriteRawData(workItem, workItem.Buffer);
}
var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]);
//MD5
workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize);
//FILE
var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash);
workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size);
Flush(); // trying to prevent bug with resized file, but no data in it
var fileSize = ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size;
if (workItem.StreamLength != fileSize)
{
workItem.ResizeStream(fileSize);
_fileSize = fileSize;
}
return(footerWithHash);
}
public static void CreateMultiChunk(TFChunkDbConfig config, int chunkStartNum, int chunkEndNum, string filename,
int?physicalSize = null, long?logicalSize = null)
{
if (chunkStartNum > chunkEndNum)
{
throw new ArgumentException("chunkStartNum");
}
var chunkHeader = new ChunkHeader(TFChunk.CurrentChunkVersion, config.ChunkSize, chunkStartNum, chunkEndNum, true, Guid.NewGuid());
var chunkBytes = chunkHeader.AsByteArray();
var physicalDataSize = physicalSize ?? config.ChunkSize;
var logicalDataSize = logicalSize ?? (chunkEndNum - chunkStartNum + 1) * config.ChunkSize;
var buf = new byte[ChunkHeader.Size + physicalDataSize + ChunkFooter.Size];
Buffer.BlockCopy(chunkBytes, 0, buf, 0, chunkBytes.Length);
var chunkFooter = new ChunkFooter(true, true, physicalDataSize, logicalDataSize, 0, new byte[ChunkFooter.ChecksumSize]);
chunkBytes = chunkFooter.AsByteArray();
Buffer.BlockCopy(chunkBytes, 0, buf, buf.Length - ChunkFooter.Size, chunkBytes.Length);
File.WriteAllBytes(filename, buf);
}
private ChunkFooter WriteFooter(ICollection<PosMap> mapping)
{
var workItem = _writerWorkItem;
int mapSize = 0;
if (mapping != null)
{
if (!_inMem && _isCached != 0)
{
throw new InvalidOperationException("Trying to write mapping while chunk is cached! "
+ "You probably are writing scavenged chunk as cached. "
+ "Don't do this!");
}
mapSize = mapping.Count * PosMap.FullSize;
workItem.Buffer.SetLength(mapSize);
workItem.Buffer.Position = 0;
foreach (var map in mapping)
{
map.Write(workItem.BufferWriter);
}
if (_inMem)
ResizeMemStream(workItem, mapSize);
WriteRawData(workItem, workItem.Buffer);
}
var footerNoHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]);
//MD5
workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize);
//FILE
var footerWithHash = new ChunkFooter(true, true, _physicalDataSize, LogicalDataSize, mapSize, workItem.MD5.Hash);
workItem.AppendData(footerWithHash.AsByteArray(), 0, ChunkFooter.Size);
Flush(); // trying to prevent bug with resized file, but no data in it
var fileSize = ChunkHeader.Size + _physicalDataSize + mapSize + ChunkFooter.Size;
if (workItem.StreamLength != fileSize)
{
workItem.ResizeStream(fileSize);
_fileSize = fileSize;
}
return footerWithHash;
}
private ChunkFooter WriteFooter(ICollection<PosMap> mapping)
{
var workItem = _writerWorkItem;
int mapSize = 0;
if (mapping != null)
{
if (_isCached != 0)
{
throw new InvalidOperationException("Trying to write mapping while chunk is cached! "
+ "You probably are writing scavenged chunk as cached. "
+ "Don't do this!");
}
mapSize = mapping.Count * sizeof(ulong);
workItem.Buffer.SetLength(mapSize);
workItem.Buffer.Position = 0;
foreach (var map in mapping)
{
workItem.BufferWriter.Write(map.AsUInt64());
}
WriteRawData(workItem, workItem.Buffer);
}
var footerNoHash = new ChunkFooter(true, _actualDataSize, _actualDataSize, mapSize, new byte[ChunkFooter.ChecksumSize]);
//MD5
workItem.MD5.TransformFinalBlock(footerNoHash.AsByteArray(), 0, ChunkFooter.Size - ChunkFooter.ChecksumSize);
//FILE
var footerWithHash = new ChunkFooter(true, _actualDataSize, _actualDataSize, mapSize, workItem.MD5.Hash);
workItem.Stream.Write(footerWithHash.AsByteArray(), 0, ChunkFooter.Size);
var fileSize = ChunkHeader.Size + _actualDataSize + mapSize + ChunkFooter.Size;
if (workItem.Stream.Length != fileSize)
workItem.Stream.SetLength(fileSize);
return footerWithHash;
}
