public void GetRequest_Batch_With_Empty_Timestamp_Generator()
{
var batch = new BatchStatement();
batch.Add(new SimpleStatement("QUERY"));
var policies = new Policies(
Policies.DefaultLoadBalancingPolicy, Policies.DefaultReconnectionPolicy, Policies.DefaultRetryPolicy,
Policies.DefaultSpeculativeExecutionPolicy, new NoTimestampGenerator());
var config = new Configuration(
policies, new ProtocolOptions(), PoolingOptions.Create(), new SocketOptions(),
new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(),
new DefaultAddressTranslator());
var request = RequestHandler.GetRequest(batch, Serializer, config);
var stream = new MemoryStream();
request.WriteFrame(1, stream, Serializer);
var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
// The batch request is composed by:
// <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>]
var offset = 1 + 2 + 1;
var queryLength = BeConverter.ToInt32(bodyBuffer, offset);
Assert.AreEqual(5, queryLength);
// skip query, n_params and consistency
offset += 4 + queryLength + 2 + 2;
var flags = (QueryFlags)bodyBuffer[offset++];
Assert.False(flags.HasFlag(QueryFlags.WithDefaultTimestamp));
// No more data
Assert.AreEqual(bodyBuffer.Length, offset);
}
private static byte[] GetBodyBuffer(IRequest request, Serializer serializer = null)
{
if (serializer == null)
{
serializer = Serializer;
}
var stream = new MemoryStream();
request.WriteFrame(1, stream, serializer);
var headerSize = FrameHeader.GetSize(serializer.ProtocolVersion);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
return(bodyBuffer);
}
public void GetRequest_With_Timestamp_Generator_Empty_Value_With_Statement_Timestamp()
{
var statement = new SimpleStatement("STATEMENT WITH TIMESTAMP");
var expectedTimestamp = new DateTimeOffset(2010, 04, 29, 1, 2, 3, 4, TimeSpan.Zero).AddTicks(20);
statement.SetTimestamp(expectedTimestamp);
var policies = new Dse.Policies(
Dse.Policies.DefaultLoadBalancingPolicy, Dse.Policies.DefaultReconnectionPolicy,
Dse.Policies.DefaultRetryPolicy, Dse.Policies.DefaultSpeculativeExecutionPolicy,
new NoTimestampGenerator());
var config = new Configuration(
policies, new ProtocolOptions(), PoolingOptions.GetDefault(ProtocolVersion.MaxSupported), new SocketOptions(), new ClientOptions(),
NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator());
var request = RequestHandler <RowSet> .GetRequest(statement, Serializer, config);
var stream = new MemoryStream();
request.WriteFrame(1, stream, Serializer);
var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
// The query request is composed by:
// <query><consistency><flags><result_page_size><timestamp>
var queryBuffer = BeConverter.GetBytes(statement.QueryString.Length)
.Concat(Encoding.UTF8.GetBytes(statement.QueryString))
.ToArray();
CollectionAssert.AreEqual(queryBuffer, bodyBuffer.Take(queryBuffer.Length));
// Skip the query and consistency (2)
var offset = queryBuffer.Length + 2;
// The remaining length should be 13 = flags (1) + result_page_size (4) + timestamp (8)
Assert.AreEqual(13, bodyBuffer.Length - offset);
var flags = (QueryFlags)bodyBuffer[offset];
Assert.True(flags.HasFlag(QueryFlags.WithDefaultTimestamp));
Assert.True(flags.HasFlag(QueryFlags.PageSize));
// Skip flags (1) + result_page_size (4)
offset += 5;
var timestamp = BeConverter.ToInt64(bodyBuffer, offset);
Assert.AreEqual(TypeSerializer.SinceUnixEpoch(expectedTimestamp).Ticks / 10, timestamp);
}
public void GetRequest_Batch_With_Timestamp_Generator()
{
var batch = new BatchStatement();
batch.Add(new SimpleStatement("QUERY"));
var startDate = DateTimeOffset.Now;
// To microsecond precision
startDate = startDate.Subtract(TimeSpan.FromTicks(startDate.Ticks % 10));
var config = new Configuration(
Policies.DefaultPolicies, new ProtocolOptions(), PoolingOptions.Create(), new SocketOptions(),
new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(),
new DefaultAddressTranslator());
var request = RequestHandler.GetRequest(batch, Serializer, config);
var stream = new MemoryStream();
request.WriteFrame(1, stream, Serializer);
var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
// The batch request is composed by:
// <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>]
var offset = 1;
// n = 1
Assert.AreEqual(1, BeConverter.ToInt16(bodyBuffer, offset));
// Query_1 <kind><string><n_params>
offset += 2;
// kind = 0, not prepared
Assert.AreEqual(0, bodyBuffer[offset++]);
var queryLength = BeConverter.ToInt32(bodyBuffer, offset);
Assert.AreEqual(5, queryLength);
// skip query, n_params and consistency
offset += 4 + queryLength + 2 + 2;
var flags = (QueryFlags)bodyBuffer[offset++];
Assert.True(flags.HasFlag(QueryFlags.WithDefaultTimestamp));
var timestamp = TypeSerializer.UnixStart.AddTicks(BeConverter.ToInt64(bodyBuffer, offset) * 10);
Assert.GreaterOrEqual(timestamp, startDate);
Assert.LessOrEqual(timestamp, DateTimeOffset.Now.Add(TimeSpan.FromMilliseconds(100)));
}
public void GetRequest_With_Timestamp_Generator()
{
// Timestamp generator should be enabled by default
var statement = new SimpleStatement("QUERY");
var config = new Configuration();
var request = RequestHandler <RowSet> .GetRequest(statement, Serializer, config);
var stream = new MemoryStream();
request.WriteFrame(1, stream, Serializer);
var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
// The query request is composed by:
// <query><consistency><flags><result_page_size><timestamp>
var queryBuffer = BeConverter.GetBytes(statement.QueryString.Length)
.Concat(Encoding.UTF8.GetBytes(statement.QueryString))
.ToArray();
CollectionAssert.AreEqual(queryBuffer, bodyBuffer.Take(queryBuffer.Length));
// Skip the query and consistency (2)
var offset = queryBuffer.Length + 2;
// The remaining length should be 13 = flags (1) + result_page_size (4) + timestamp (8)
Assert.AreEqual(13, bodyBuffer.Length - offset);
var flags = (QueryFlags)bodyBuffer[offset];
Assert.True(flags.HasFlag(QueryFlags.WithDefaultTimestamp));
Assert.True(flags.HasFlag(QueryFlags.PageSize));
Assert.False(flags.HasFlag(QueryFlags.Values));
Assert.False(flags.HasFlag(QueryFlags.WithPagingState));
Assert.False(flags.HasFlag(QueryFlags.SkipMetadata));
Assert.False(flags.HasFlag(QueryFlags.WithSerialConsistency));
// Skip flags (1) + result_page_size (4)
offset += 5;
var timestamp = BeConverter.ToInt64(bodyBuffer, offset);
var expectedTimestamp = TypeSerializer.SinceUnixEpoch(DateTimeOffset.Now.Subtract(TimeSpan.FromMilliseconds(100))).Ticks / 10;
Assert.Greater(timestamp, expectedTimestamp);
}
public void GetRequest_With_Timestamp_Generator_Empty_Value()
{
var statement = new SimpleStatement("QUERY");
var policies = new Dse.Policies(
Dse.Policies.DefaultLoadBalancingPolicy, Dse.Policies.DefaultReconnectionPolicy,
Dse.Policies.DefaultRetryPolicy, Dse.Policies.DefaultSpeculativeExecutionPolicy,
new NoTimestampGenerator());
var config = new Configuration(
policies, new ProtocolOptions(), PoolingOptions.GetDefault(ProtocolVersion.MaxSupported), new SocketOptions(), new ClientOptions(),
NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator());
var request = RequestHandler <RowSet> .GetRequest(statement, Serializer.Default, config);
var stream = new MemoryStream();
request.WriteFrame(1, stream, Serializer);
var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
// The query request is composed by:
// <query><consistency><flags><result_page_size>
var queryBuffer = BeConverter.GetBytes(statement.QueryString.Length)
.Concat(Encoding.UTF8.GetBytes(statement.QueryString))
.ToArray();
CollectionAssert.AreEqual(queryBuffer, bodyBuffer.Take(queryBuffer.Length));
// Skip the query and consistency (2)
var offset = queryBuffer.Length + 2;
// The remaining length should be 13 = flags (1) + result_page_size (4)
Assert.AreEqual(5, bodyBuffer.Length - offset);
var flags = (QueryFlags)bodyBuffer[offset];
Assert.False(flags.HasFlag(QueryFlags.WithDefaultTimestamp));
Assert.True(flags.HasFlag(QueryFlags.PageSize));
Assert.False(flags.HasFlag(QueryFlags.Values));
Assert.False(flags.HasFlag(QueryFlags.WithPagingState));
Assert.False(flags.HasFlag(QueryFlags.SkipMetadata));
Assert.False(flags.HasFlag(QueryFlags.WithSerialConsistency));
}
public void GetRequest_Batch_With_64K_Queries()
{
var batch = new BatchStatement();
for (var i = 0; i < ushort.MaxValue; i++)
{
batch.Add(new SimpleStatement("QUERY"));
}
var config = new Configuration(
Policies.DefaultPolicies, new ProtocolOptions(), PoolingOptions.Create(), new SocketOptions(),
new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator());
var request = RequestHandler.GetRequest(batch, Serializer, config);
var stream = new MemoryStream();
request.WriteFrame(1, stream, Serializer);
var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported);
var bodyBuffer = new byte[stream.Length - headerSize];
stream.Position = headerSize;
stream.Read(bodyBuffer, 0, bodyBuffer.Length);
// The batch request is composed by:
// <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>]
CollectionAssert.AreEqual(new byte[] { 0xff, 0xff }, bodyBuffer.Skip(1).Take(2));
}
/// <summary>
/// Deserializes each frame header and copies the body bytes into a single buffer.
/// </summary>
/// <returns>True if a full operation (streamId) has been processed.</returns>
internal bool ReadParse(byte[] buffer, int length)
{
if (length <= 0)
{
return(false);
}
ProtocolVersion protocolVersion;
var headerLength = Volatile.Read(ref _frameHeaderSize);
if (headerLength == 0)
{
// The server replies the first message with the max protocol version supported
protocolVersion = FrameHeader.GetProtocolVersion(buffer);
_serializer.ProtocolVersion = protocolVersion;
headerLength = FrameHeader.GetSize(protocolVersion);
Volatile.Write(ref _frameHeaderSize, headerLength);
}
else
{
protocolVersion = _serializer.ProtocolVersion;
}
// Use _readStream to buffer between messages, when the body is not contained in a single read call
var stream = Interlocked.Exchange(ref _readStream, null);
var previousHeader = Interlocked.Exchange(ref _receivingHeader, null);
if (previousHeader != null && stream == null)
{
// This connection has been disposed
return(false);
}
var operationCallbacks = new LinkedList <Action <MemoryStream> >();
var offset = 0;
while (offset < length)
{
var header = previousHeader;
int remainingBodyLength;
if (header == null)
{
header = ReadHeader(buffer, ref offset, length, headerLength, protocolVersion);
if (header == null)
{
// There aren't enough bytes to read the header
break;
}
Connection.Logger.Verbose("Received #{0} from {1}", header.StreamId, Address);
remainingBodyLength = header.BodyLength;
}
else
{
previousHeader = null;
remainingBodyLength = header.BodyLength - (int)stream.Length;
}
if (remainingBodyLength > length - offset)
{
// The buffer does not contains the body for the current frame, store it for later
StoreReadState(header, stream, buffer, offset, length, operationCallbacks.Count > 0);
break;
}
stream = stream ?? Configuration.BufferPool.GetStream(Connection.StreamReadTag);
var state = header.Opcode != EventResponse.OpCode
? RemoveFromPending(header.StreamId)
: new OperationState(EventHandler);
stream.Write(buffer, offset, remainingBodyLength);
// State can be null when the Connection is being closed concurrently
// The original callback is being called with an error, use a Noop here
var callback = state != null?state.SetCompleted() : OperationState.Noop;
operationCallbacks.AddLast(CreateResponseAction(header, callback));
offset += remainingBodyLength;
}
return(Connection.InvokeReadCallbacks(stream, operationCallbacks));
}
public virtual void DemultiplexStreams(MpegStream.DemuxOptionsStruct demuxOptions)
{
long currentOffset = -1;
long fileSize;
byte[] dummy;
uint blocksProcessed = 0;
uint audioFramesProcessed = 0;
uint videoFramesProcessed = 0;
uint bytesProcessed = 0;
uint expectedBytesProcessed = 0;
BlockHeader blockHeader = new BlockHeader();
FrameHeader frameHeader = new FrameHeader();
bool isFirstVideoFrame = true;
bool isFirstAudioFrame = true;
byte[] fullChunk;
uint audioChunkSize;
uint audioChunkSamples;
long blockStart;
long frameStart;
Dictionary <string, FileStream> streamWriters = new Dictionary <string, FileStream>();
try
{
using (FileStream fs = File.OpenRead(this.FilePath))
{
fileSize = fs.Length;
#region HEADER CHECK
// check header
dummy = ParseFile.ParseSimpleOffset(fs, 0, 0x10);
if (ParseFile.CompareSegment(dummy, 0, HVQM4_13_SIGNATURE))
{
this.FileRevision = VersionType.HVQM4_13;
currentOffset = 0;
}
else if (ParseFile.CompareSegment(dummy, 0, HVQM4_15_SIGNATURE))
{
this.FileRevision = VersionType.HVQM4_15;
currentOffset = 0;
}
else
{
throw new Exception("HVQM4 signature not found at offset 0x00");
}
#endregion
// parse file
if (currentOffset >= 0)
{
// get header
this.ParseHeader(fs, 0);
currentOffset = this.HeaderSize + 4;
// process file
while ((currentOffset < fileSize) &&
(blocksProcessed < this.Blocks))
{
// reset flags
isFirstVideoFrame = true;
isFirstAudioFrame = true;
audioFramesProcessed = 0;
videoFramesProcessed = 0;
//--------------
// parse block
//--------------
blockStart = currentOffset;
blocksProcessed++;
// parse block header
this.ParseBlockHeader(fs, currentOffset, ref blockHeader);
currentOffset += blockHeader.GetSize();
bytesProcessed = 0;
while (bytesProcessed < blockHeader.BlockSize)
{
frameStart = currentOffset;
// verify we haven't processed too much
if (audioFramesProcessed > blockHeader.AudioFrameCount)
{
throw new Exception(String.Format("Processed more audio frames than expected for block starting at 0x{0}", blockStart.ToString("X8")));
}
else if (videoFramesProcessed > blockHeader.VideoFrameCount)
{
throw new Exception(String.Format("Processed more video frames than expected for block starting at 0x{0}", blockStart.ToString("X8")));
}
// parse frame header
this.ParseFrameHeader(fs, currentOffset, ref frameHeader);
currentOffset += frameHeader.GetSize();
bytesProcessed += (uint)frameHeader.GetSize();
//---------------
// process frame
//---------------
// audio chunk
if (demuxOptions.ExtractAudio && frameHeader.IsAudioFrame(isFirstAudioFrame))
{
audioChunkSamples = ParseFile.ReadUintBE(fs, currentOffset);
audioChunkSize = (audioChunkSamples * this.AudioChannels) / 2;
// get full frame for now
fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset, (int)frameHeader.FrameSize);
// different frames have different IMA info
//if (isFirstAudioFrame)
//{
// fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset + 6, (int)audioChunkSize);
//}
//else
//{
// fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset + 4, (int)audioChunkSize);
//}
this.writeChunkToStream(fullChunk, "audio", streamWriters, this.FileExtensionAudio);
isFirstAudioFrame = false;
audioFramesProcessed++;
}
// video chunk
else if (demuxOptions.ExtractVideo && frameHeader.IsVideoFrame(isFirstVideoFrame, this.FileRevision))
{
fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset, (int)frameHeader.FrameSize);
this.writeChunkToStream(fullChunk, "video", streamWriters, this.FileExtensionVideo);
isFirstVideoFrame = false;
videoFramesProcessed++;
}
//unknown
else
{
fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset, (int)frameHeader.FrameSize);
this.writeChunkToStream(fullChunk, "unknown", streamWriters, ".bin");
}
// update number of bytes processed
bytesProcessed += frameHeader.FrameSize;
// move to next frame
currentOffset += frameHeader.FrameSize;
} // while (bytesProcessed < blockHeader.BlockSize)
// verify proper number of bytes processed
if (blocksProcessed < this.Blocks)
{
bytesProcessed = bytesProcessed + (uint)(blockHeader.GetSize() + 4);
expectedBytesProcessed = ParseFile.ReadUintBE(fs, currentOffset);
if (expectedBytesProcessed != bytesProcessed)
{
throw new Exception(String.Format(
"Bytes processed 0x{0}does not match expected bytes processed {1} for block starting at 0x{2}.",
bytesProcessed.ToString("X8"),
expectedBytesProcessed.ToString("X8"),
blockStart.ToString("X8")));
}
else
{
currentOffset += 4;
}
}
}
}
else
{
throw new Exception(String.Format("Cannot find MVHD header.{0}", Environment.NewLine));
}
}
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
finally
{
this.DoFinalTasks(streamWriters);
}
}
