public void GetRequest_Batch_With_Empty_Timestamp_Generator() { var batch = new BatchStatement(); batch.Add(new SimpleStatement("QUERY")); var policies = new Policies( Policies.DefaultLoadBalancingPolicy, Policies.DefaultReconnectionPolicy, Policies.DefaultRetryPolicy, Policies.DefaultSpeculativeExecutionPolicy, new NoTimestampGenerator()); var config = new Configuration( policies, new ProtocolOptions(), PoolingOptions.Create(), new SocketOptions(), new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator()); var request = RequestHandler.GetRequest(batch, Serializer, config); var stream = new MemoryStream(); request.WriteFrame(1, stream, Serializer); var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); // The batch request is composed by: // <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>] var offset = 1 + 2 + 1; var queryLength = BeConverter.ToInt32(bodyBuffer, offset); Assert.AreEqual(5, queryLength); // skip query, n_params and consistency offset += 4 + queryLength + 2 + 2; var flags = (QueryFlags)bodyBuffer[offset++]; Assert.False(flags.HasFlag(QueryFlags.WithDefaultTimestamp)); // No more data Assert.AreEqual(bodyBuffer.Length, offset); }
private static byte[] GetBodyBuffer(IRequest request, Serializer serializer = null) { if (serializer == null) { serializer = Serializer; } var stream = new MemoryStream(); request.WriteFrame(1, stream, serializer); var headerSize = FrameHeader.GetSize(serializer.ProtocolVersion); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); return(bodyBuffer); }
public void GetRequest_With_Timestamp_Generator_Empty_Value_With_Statement_Timestamp() { var statement = new SimpleStatement("STATEMENT WITH TIMESTAMP"); var expectedTimestamp = new DateTimeOffset(2010, 04, 29, 1, 2, 3, 4, TimeSpan.Zero).AddTicks(20); statement.SetTimestamp(expectedTimestamp); var policies = new Dse.Policies( Dse.Policies.DefaultLoadBalancingPolicy, Dse.Policies.DefaultReconnectionPolicy, Dse.Policies.DefaultRetryPolicy, Dse.Policies.DefaultSpeculativeExecutionPolicy, new NoTimestampGenerator()); var config = new Configuration( policies, new ProtocolOptions(), PoolingOptions.GetDefault(ProtocolVersion.MaxSupported), new SocketOptions(), new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator()); var request = RequestHandler <RowSet> .GetRequest(statement, Serializer, config); var stream = new MemoryStream(); request.WriteFrame(1, stream, Serializer); var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); // The query request is composed by: // <query><consistency><flags><result_page_size><timestamp> var queryBuffer = BeConverter.GetBytes(statement.QueryString.Length) .Concat(Encoding.UTF8.GetBytes(statement.QueryString)) .ToArray(); CollectionAssert.AreEqual(queryBuffer, bodyBuffer.Take(queryBuffer.Length)); // Skip the query and consistency (2) var offset = queryBuffer.Length + 2; // The remaining length should be 13 = flags (1) + result_page_size (4) + timestamp (8) Assert.AreEqual(13, bodyBuffer.Length - offset); var flags = (QueryFlags)bodyBuffer[offset]; Assert.True(flags.HasFlag(QueryFlags.WithDefaultTimestamp)); Assert.True(flags.HasFlag(QueryFlags.PageSize)); // Skip flags (1) + result_page_size (4) offset += 5; var timestamp = BeConverter.ToInt64(bodyBuffer, offset); Assert.AreEqual(TypeSerializer.SinceUnixEpoch(expectedTimestamp).Ticks / 10, timestamp); }
public void GetRequest_Batch_With_Timestamp_Generator() { var batch = new BatchStatement(); batch.Add(new SimpleStatement("QUERY")); var startDate = DateTimeOffset.Now; // To microsecond precision startDate = startDate.Subtract(TimeSpan.FromTicks(startDate.Ticks % 10)); var config = new Configuration( Policies.DefaultPolicies, new ProtocolOptions(), PoolingOptions.Create(), new SocketOptions(), new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator()); var request = RequestHandler.GetRequest(batch, Serializer, config); var stream = new MemoryStream(); request.WriteFrame(1, stream, Serializer); var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); // The batch request is composed by: // <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>] var offset = 1; // n = 1 Assert.AreEqual(1, BeConverter.ToInt16(bodyBuffer, offset)); // Query_1 <kind><string><n_params> offset += 2; // kind = 0, not prepared Assert.AreEqual(0, bodyBuffer[offset++]); var queryLength = BeConverter.ToInt32(bodyBuffer, offset); Assert.AreEqual(5, queryLength); // skip query, n_params and consistency offset += 4 + queryLength + 2 + 2; var flags = (QueryFlags)bodyBuffer[offset++]; Assert.True(flags.HasFlag(QueryFlags.WithDefaultTimestamp)); var timestamp = TypeSerializer.UnixStart.AddTicks(BeConverter.ToInt64(bodyBuffer, offset) * 10); Assert.GreaterOrEqual(timestamp, startDate); Assert.LessOrEqual(timestamp, DateTimeOffset.Now.Add(TimeSpan.FromMilliseconds(100))); }
public void GetRequest_With_Timestamp_Generator() { // Timestamp generator should be enabled by default var statement = new SimpleStatement("QUERY"); var config = new Configuration(); var request = RequestHandler <RowSet> .GetRequest(statement, Serializer, config); var stream = new MemoryStream(); request.WriteFrame(1, stream, Serializer); var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); // The query request is composed by: // <query><consistency><flags><result_page_size><timestamp> var queryBuffer = BeConverter.GetBytes(statement.QueryString.Length) .Concat(Encoding.UTF8.GetBytes(statement.QueryString)) .ToArray(); CollectionAssert.AreEqual(queryBuffer, bodyBuffer.Take(queryBuffer.Length)); // Skip the query and consistency (2) var offset = queryBuffer.Length + 2; // The remaining length should be 13 = flags (1) + result_page_size (4) + timestamp (8) Assert.AreEqual(13, bodyBuffer.Length - offset); var flags = (QueryFlags)bodyBuffer[offset]; Assert.True(flags.HasFlag(QueryFlags.WithDefaultTimestamp)); Assert.True(flags.HasFlag(QueryFlags.PageSize)); Assert.False(flags.HasFlag(QueryFlags.Values)); Assert.False(flags.HasFlag(QueryFlags.WithPagingState)); Assert.False(flags.HasFlag(QueryFlags.SkipMetadata)); Assert.False(flags.HasFlag(QueryFlags.WithSerialConsistency)); // Skip flags (1) + result_page_size (4) offset += 5; var timestamp = BeConverter.ToInt64(bodyBuffer, offset); var expectedTimestamp = TypeSerializer.SinceUnixEpoch(DateTimeOffset.Now.Subtract(TimeSpan.FromMilliseconds(100))).Ticks / 10; Assert.Greater(timestamp, expectedTimestamp); }
public void GetRequest_With_Timestamp_Generator_Empty_Value() { var statement = new SimpleStatement("QUERY"); var policies = new Dse.Policies( Dse.Policies.DefaultLoadBalancingPolicy, Dse.Policies.DefaultReconnectionPolicy, Dse.Policies.DefaultRetryPolicy, Dse.Policies.DefaultSpeculativeExecutionPolicy, new NoTimestampGenerator()); var config = new Configuration( policies, new ProtocolOptions(), PoolingOptions.GetDefault(ProtocolVersion.MaxSupported), new SocketOptions(), new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator()); var request = RequestHandler <RowSet> .GetRequest(statement, Serializer.Default, config); var stream = new MemoryStream(); request.WriteFrame(1, stream, Serializer); var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); // The query request is composed by: // <query><consistency><flags><result_page_size> var queryBuffer = BeConverter.GetBytes(statement.QueryString.Length) .Concat(Encoding.UTF8.GetBytes(statement.QueryString)) .ToArray(); CollectionAssert.AreEqual(queryBuffer, bodyBuffer.Take(queryBuffer.Length)); // Skip the query and consistency (2) var offset = queryBuffer.Length + 2; // The remaining length should be 13 = flags (1) + result_page_size (4) Assert.AreEqual(5, bodyBuffer.Length - offset); var flags = (QueryFlags)bodyBuffer[offset]; Assert.False(flags.HasFlag(QueryFlags.WithDefaultTimestamp)); Assert.True(flags.HasFlag(QueryFlags.PageSize)); Assert.False(flags.HasFlag(QueryFlags.Values)); Assert.False(flags.HasFlag(QueryFlags.WithPagingState)); Assert.False(flags.HasFlag(QueryFlags.SkipMetadata)); Assert.False(flags.HasFlag(QueryFlags.WithSerialConsistency)); }
public void GetRequest_Batch_With_64K_Queries() { var batch = new BatchStatement(); for (var i = 0; i < ushort.MaxValue; i++) { batch.Add(new SimpleStatement("QUERY")); } var config = new Configuration( Policies.DefaultPolicies, new ProtocolOptions(), PoolingOptions.Create(), new SocketOptions(), new ClientOptions(), NoneAuthProvider.Instance, null, new QueryOptions(), new DefaultAddressTranslator()); var request = RequestHandler.GetRequest(batch, Serializer, config); var stream = new MemoryStream(); request.WriteFrame(1, stream, Serializer); var headerSize = FrameHeader.GetSize(ProtocolVersion.MaxSupported); var bodyBuffer = new byte[stream.Length - headerSize]; stream.Position = headerSize; stream.Read(bodyBuffer, 0, bodyBuffer.Length); // The batch request is composed by: // <type><n><query_1>...<query_n><consistency><flags>[<serial_consistency>][<timestamp>] CollectionAssert.AreEqual(new byte[] { 0xff, 0xff }, bodyBuffer.Skip(1).Take(2)); }
/// <summary> /// Deserializes each frame header and copies the body bytes into a single buffer. /// </summary> /// <returns>True if a full operation (streamId) has been processed.</returns> internal bool ReadParse(byte[] buffer, int length) { if (length <= 0) { return(false); } ProtocolVersion protocolVersion; var headerLength = Volatile.Read(ref _frameHeaderSize); if (headerLength == 0) { // The server replies the first message with the max protocol version supported protocolVersion = FrameHeader.GetProtocolVersion(buffer); _serializer.ProtocolVersion = protocolVersion; headerLength = FrameHeader.GetSize(protocolVersion); Volatile.Write(ref _frameHeaderSize, headerLength); } else { protocolVersion = _serializer.ProtocolVersion; } // Use _readStream to buffer between messages, when the body is not contained in a single read call var stream = Interlocked.Exchange(ref _readStream, null); var previousHeader = Interlocked.Exchange(ref _receivingHeader, null); if (previousHeader != null && stream == null) { // This connection has been disposed return(false); } var operationCallbacks = new LinkedList <Action <MemoryStream> >(); var offset = 0; while (offset < length) { var header = previousHeader; int remainingBodyLength; if (header == null) { header = ReadHeader(buffer, ref offset, length, headerLength, protocolVersion); if (header == null) { // There aren't enough bytes to read the header break; } Connection.Logger.Verbose("Received #{0} from {1}", header.StreamId, Address); remainingBodyLength = header.BodyLength; } else { previousHeader = null; remainingBodyLength = header.BodyLength - (int)stream.Length; } if (remainingBodyLength > length - offset) { // The buffer does not contains the body for the current frame, store it for later StoreReadState(header, stream, buffer, offset, length, operationCallbacks.Count > 0); break; } stream = stream ?? Configuration.BufferPool.GetStream(Connection.StreamReadTag); var state = header.Opcode != EventResponse.OpCode ? RemoveFromPending(header.StreamId) : new OperationState(EventHandler); stream.Write(buffer, offset, remainingBodyLength); // State can be null when the Connection is being closed concurrently // The original callback is being called with an error, use a Noop here var callback = state != null?state.SetCompleted() : OperationState.Noop; operationCallbacks.AddLast(CreateResponseAction(header, callback)); offset += remainingBodyLength; } return(Connection.InvokeReadCallbacks(stream, operationCallbacks)); }
public virtual void DemultiplexStreams(MpegStream.DemuxOptionsStruct demuxOptions) { long currentOffset = -1; long fileSize; byte[] dummy; uint blocksProcessed = 0; uint audioFramesProcessed = 0; uint videoFramesProcessed = 0; uint bytesProcessed = 0; uint expectedBytesProcessed = 0; BlockHeader blockHeader = new BlockHeader(); FrameHeader frameHeader = new FrameHeader(); bool isFirstVideoFrame = true; bool isFirstAudioFrame = true; byte[] fullChunk; uint audioChunkSize; uint audioChunkSamples; long blockStart; long frameStart; Dictionary <string, FileStream> streamWriters = new Dictionary <string, FileStream>(); try { using (FileStream fs = File.OpenRead(this.FilePath)) { fileSize = fs.Length; #region HEADER CHECK // check header dummy = ParseFile.ParseSimpleOffset(fs, 0, 0x10); if (ParseFile.CompareSegment(dummy, 0, HVQM4_13_SIGNATURE)) { this.FileRevision = VersionType.HVQM4_13; currentOffset = 0; } else if (ParseFile.CompareSegment(dummy, 0, HVQM4_15_SIGNATURE)) { this.FileRevision = VersionType.HVQM4_15; currentOffset = 0; } else { throw new Exception("HVQM4 signature not found at offset 0x00"); } #endregion // parse file if (currentOffset >= 0) { // get header this.ParseHeader(fs, 0); currentOffset = this.HeaderSize + 4; // process file while ((currentOffset < fileSize) && (blocksProcessed < this.Blocks)) { // reset flags isFirstVideoFrame = true; isFirstAudioFrame = true; audioFramesProcessed = 0; videoFramesProcessed = 0; //-------------- // parse block //-------------- blockStart = currentOffset; blocksProcessed++; // parse block header this.ParseBlockHeader(fs, currentOffset, ref blockHeader); currentOffset += blockHeader.GetSize(); bytesProcessed = 0; while (bytesProcessed < blockHeader.BlockSize) { frameStart = currentOffset; // verify we haven't processed too much if (audioFramesProcessed > blockHeader.AudioFrameCount) { throw new Exception(String.Format("Processed more audio frames than expected for block starting at 0x{0}", blockStart.ToString("X8"))); } else if (videoFramesProcessed > blockHeader.VideoFrameCount) { throw new Exception(String.Format("Processed more video frames than expected for block starting at 0x{0}", blockStart.ToString("X8"))); } // parse frame header this.ParseFrameHeader(fs, currentOffset, ref frameHeader); currentOffset += frameHeader.GetSize(); bytesProcessed += (uint)frameHeader.GetSize(); //--------------- // process frame //--------------- // audio chunk if (demuxOptions.ExtractAudio && frameHeader.IsAudioFrame(isFirstAudioFrame)) { audioChunkSamples = ParseFile.ReadUintBE(fs, currentOffset); audioChunkSize = (audioChunkSamples * this.AudioChannels) / 2; // get full frame for now fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset, (int)frameHeader.FrameSize); // different frames have different IMA info //if (isFirstAudioFrame) //{ // fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset + 6, (int)audioChunkSize); //} //else //{ // fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset + 4, (int)audioChunkSize); //} this.writeChunkToStream(fullChunk, "audio", streamWriters, this.FileExtensionAudio); isFirstAudioFrame = false; audioFramesProcessed++; } // video chunk else if (demuxOptions.ExtractVideo && frameHeader.IsVideoFrame(isFirstVideoFrame, this.FileRevision)) { fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset, (int)frameHeader.FrameSize); this.writeChunkToStream(fullChunk, "video", streamWriters, this.FileExtensionVideo); isFirstVideoFrame = false; videoFramesProcessed++; } //unknown else { fullChunk = ParseFile.ParseSimpleOffset(fs, currentOffset, (int)frameHeader.FrameSize); this.writeChunkToStream(fullChunk, "unknown", streamWriters, ".bin"); } // update number of bytes processed bytesProcessed += frameHeader.FrameSize; // move to next frame currentOffset += frameHeader.FrameSize; } // while (bytesProcessed < blockHeader.BlockSize) // verify proper number of bytes processed if (blocksProcessed < this.Blocks) { bytesProcessed = bytesProcessed + (uint)(blockHeader.GetSize() + 4); expectedBytesProcessed = ParseFile.ReadUintBE(fs, currentOffset); if (expectedBytesProcessed != bytesProcessed) { throw new Exception(String.Format( "Bytes processed 0x{0}does not match expected bytes processed {1} for block starting at 0x{2}.", bytesProcessed.ToString("X8"), expectedBytesProcessed.ToString("X8"), blockStart.ToString("X8"))); } else { currentOffset += 4; } } } } else { throw new Exception(String.Format("Cannot find MVHD header.{0}", Environment.NewLine)); } } } catch (Exception ex) { throw new Exception(ex.Message); } finally { this.DoFinalTasks(streamWriters); } }