private void VerifyTargetStreamsAreComplete(TransferMetadata metadata, InMemoryFrontEnd fe)
{
foreach (var segment in metadata.Segments)
{
VerifyTargetStreamIsComplete(segment, metadata, fe);
}
}
private TransferMetadata CreateMetadata(int segmentCount)
{
var path = Path.GetTempFileName();
var metadata = new TransferMetadata()
{
MetadataFilePath = path,
InputFilePath = _smallFilePath,
FileLength = _smallFileContents.Length,
SegmentCount = segmentCount,
SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(_smallFileContents.Length, segmentCount),
Segments = new TransferSegmentMetadata[segmentCount],
TargetStreamPath = "abc",
TransferId = "123",
IsBinary = true
};
long offset = 0;
for (int i = 0; i < segmentCount; i++)
{
long length = TransferSegmentMetadata.CalculateSegmentLength(i, metadata);
metadata.Segments[i] = new TransferSegmentMetadata()
{
SegmentNumber = i,
Offset = offset,
Status = SegmentTransferStatus.Pending,
Length = length,
Path = string.Format("{0}.{1}.segment{2}", metadata.TargetStreamPath, metadata.TransferId, i)
};
offset += length;
}
return(metadata);
}
private void VerifySegmentsAreOnRecordBoundaries(TransferMetadata metadata, string filePath)
{
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
foreach (var segment in metadata.Segments)
{
if (segment.SegmentNumber > 0)
{
//verify that each segment starts with a non-newline and that the 2 previous characters before that offset are newline characters
//2 characters behind: newline
stream.Seek(segment.Offset - 2, SeekOrigin.Begin);
char c1 = (char)stream.ReadByte();
Assert.True(IsNewline(c1), string.Format("Expecting a newline at offset {0}", stream.Position - 1));
//1 character behind: newline
char c2 = (char)stream.ReadByte();
Assert.True(IsNewline(c2), string.Format("Expecting a newline at offset {0}", stream.Position - 1));
//by test design, we never have two consecutive newlines that are the same; we'd always have \r\n, but never \r\r or \r\n
var c3 = (char)stream.ReadByte();
Assert.NotEqual(c2, c3);
}
}
}
}
public void UploadSegmentMetadata_CalculateParticularSegmentLength()
{
//verify bad inputs
Assert.Throws <ArgumentOutOfRangeException>(
() => { TransferSegmentMetadata.CalculateSegmentLength(-1, new TransferMetadata()
{
FileLength = 10, SegmentCount = 5, SegmentLength = 2
}); });
Assert.Throws <ArgumentOutOfRangeException>(
() => { TransferSegmentMetadata.CalculateSegmentLength(100, new TransferMetadata()
{
FileLength = 10, SegmentCount = 5, SegmentLength = 2
}); });
Assert.Throws <ArgumentException>(
() => { TransferSegmentMetadata.CalculateSegmentLength(1, new TransferMetadata()
{
FileLength = -10, SegmentCount = 5, SegmentLength = 2
}); });
Assert.Throws <ArgumentException>(
() => { TransferSegmentMetadata.CalculateSegmentLength(1, new TransferMetadata()
{
FileLength = 100, SegmentCount = 2, SegmentLength = 2
}); });
Assert.Throws <ArgumentException>(
() => { TransferSegmentMetadata.CalculateSegmentLength(1, new TransferMetadata()
{
FileLength = 100, SegmentCount = 5, SegmentLength = 26
}); });
//test various scenarios with a fixed file length, and varying the segment count from 1 to the FileLength
int FileLength = 16 * (int)Math.Pow(2, 20);//16MB
for (int segmentCount = 1; segmentCount <= FileLength; segmentCount += 1024)
{
long typicalSegmentLength = TransferSegmentMetadata.CalculateSegmentLength(FileLength, segmentCount);
var uploadMetadata = new TransferMetadata()
{
FileLength = FileLength, SegmentCount = segmentCount, SegmentLength = typicalSegmentLength
};
long firstSegmentLength = TransferSegmentMetadata.CalculateSegmentLength(0, uploadMetadata);
long lastSegmentLength = TransferSegmentMetadata.CalculateSegmentLength(segmentCount - 1, uploadMetadata);
Assert.Equal(typicalSegmentLength, firstSegmentLength);
if (segmentCount == 1)
{
Assert.Equal(firstSegmentLength, lastSegmentLength);
}
long reconstructedFileLength = typicalSegmentLength * (segmentCount - 1) + lastSegmentLength;
Assert.Equal(FileLength, reconstructedFileLength);
}
}
private TransferMetadata CreateMetadata(string filePath, long fileLength)
{
var metadata = new TransferMetadata()
{
InputFilePath = filePath,
FileLength = fileLength,
TargetStreamPath = StreamPath,
SegmentCount = 1,
SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(fileLength, 1),
Segments = new TransferSegmentMetadata[1],
IsBinary = true
};
metadata.Segments[0] = new TransferSegmentMetadata(0, metadata);
metadata.Segments[0].Path = metadata.TargetStreamPath;
return(metadata);
}
private byte[] GetExpectedContents(TransferSegmentMetadata segment, TransferMetadata metadata)
{
byte[] result = new byte[segment.Length];
Array.Copy(_smallFileContents, (int)(segment.SegmentNumber * metadata.SegmentLength), result, 0, (int)segment.Length);
return(result);
}
private void VerifyTargetStreamIsComplete(TransferSegmentMetadata segmentMetadata, TransferMetadata metadata, InMemoryFrontEnd frontEnd)
{
Assert.Equal(SegmentTransferStatus.Complete, segmentMetadata.Status);
Assert.True(frontEnd.StreamExists(segmentMetadata.Path), string.Format("Segment {0} was not uploaded", segmentMetadata.SegmentNumber));
Assert.Equal(segmentMetadata.Length, frontEnd.GetStreamLength(segmentMetadata.Path));
var actualContents = frontEnd.GetStreamContents(segmentMetadata.Path);
var expectedContents = GetExpectedContents(segmentMetadata, metadata);
AssertExtensions.AreEqual(expectedContents, actualContents, "Segment {0} has unexpected contents", segmentMetadata.SegmentNumber);
}