public SplitKeyRangeRequest(
string tableId,
byte[] startPrimaryKey,
byte[] endPrimaryKey,
byte[] partitionKey,
long splitSizeBytes)
{
_request = new SplitKeyRangeRequestPB
{
TargetChunkSizeBytes = (ulong)splitSizeBytes
};
if (startPrimaryKey != null && startPrimaryKey.Length > 0)
{
_request.StartPrimaryKey = UnsafeByteOperations.UnsafeWrap(startPrimaryKey);
}
if (endPrimaryKey != null && endPrimaryKey.Length > 0)
{
_request.StopPrimaryKey = UnsafeByteOperations.UnsafeWrap(endPrimaryKey);
}
MethodName = "SplitKeyRange";
TableId = tableId;
PartitionKey = partitionKey;
NeedsAuthzToken = true;
}
public async Task Dispatch_Command_ReplyReturned()
{
var serverCallContext = TestServerCallContextFactory.Create();
serverCallContext.UserState["__HttpContext"] = httpContextFactory();
var dispatcher = new DispatcherService();
var cmd = new TestCommand
{
Value = Guid.NewGuid().ToString()
};
var request = new RequestEnvelope()
{
Type = cmd.GetType().AssemblyQualifiedName,
Data = UnsafeByteOperations.UnsafeWrap(JsonSerializer.SerializeToUtf8Bytes(cmd))
};
var response = await dispatcher.Dispatch(request, serverCallContext);
response.ShouldNotBeNull().Data.ShouldNotBeNull().ShouldNotBeEmpty();
var responseType = System.Type.GetType(response.Type, an => Assembly.Load(an.Name ?? null !), null, true, true).ShouldNotBeNull();
using var ms = new MemoryStream(response.Data.ToByteArray());
JsonSerializer.Deserialize(ms, responseType).ShouldBeOfType <string>().ShouldBe(cmd.Value);
}
public static async Task <TReply?> DispatchAsync <TReply>(this Dispatcher.DispatcherClient dispatcherClient, object content)
{
var request = new RequestEnvelope
{
CorrelationId = Guid.NewGuid().ToString(),
Type = content.GetType().AssemblyQualifiedName,
Data = UnsafeByteOperations.UnsafeWrap(JsonSerializer.SerializeToUtf8Bytes(content))
};
var response = await dispatcherClient.DispatchAsync(request, new CallOptions(deadline : DateTime.UtcNow.AddSeconds(118), headers : new Metadata()));
if (response.Error)
{
var errorType = Type.GetType(response.ErrorType, an => Assembly.Load(an.Name ?? null !), null, false, true);
var exception = errorType != null
? (Exception)(Activator.CreateInstance(errorType, response.ErrorMessage) ?? null !)
: new Exception(response.ErrorMessage);
throw exception;
//var serializer = new System.Runtime.Serialization.DataContractSerializer(errorType);
//using var ms = new MemoryStream(response.Data.ToByteArray());
//ms.Position = 0;
//var reader = XmlDictionaryReader.CreateTextReader(ms, XmlDictionaryReaderQuotas.Max);
//var exception = (Exception)(serializer.ReadObject(reader) ?? null!);
//throw exception;
}
else
{
if (response.Empty || string.IsNullOrEmpty(response.Type))
{
return(default);
/// <summary>
/// Change the default value for a column. `newDefault` must not be null or
/// else throws.
/// </summary>
/// <param name="name">Name of the column.</param>
/// <param name="newDefault">The new default value.</param>
public AlterTableBuilder ChangeDefault(string name, object newDefault)
{
if (newDefault is null)
{
ThrowDefaultValueNullException();
}
var column = _table.Schema.GetColumn(name);
var defaultValue = KuduEncoder.EncodeDefaultValue(column, newDefault);
_request.AlterSchemaSteps.Add(new Step
{
Type = StepType.AlterColumn,
AlterColumn = new AlterColumn
{
Delta = new ColumnSchemaDeltaPB
{
Name = name,
DefaultValue = UnsafeByteOperations.UnsafeWrap(defaultValue)
}
}
});
return(this);
}
public override async Task StreamingBothWays(IAsyncStreamReader <SimpleRequest> requestStream, IServerStreamWriter <SimpleResponse> responseStream, ServerCallContext context)
{
var response = new SimpleResponse
{
Payload = new Payload {
Body = UnsafeByteOperations.UnsafeWrap(new byte[100])
}
};
var clientComplete = false;
var readTask = Task.Run(async() =>
{
await foreach (var message in requestStream.ReadAllAsync())
{
// Nom nom nom
}
clientComplete = true;
});
// Write outgoing messages until client is complete
while (!clientComplete)
{
await responseStream.WriteAsync(response);
}
await readTask;
}
public static SimpleResponse CreateResponse(SimpleRequest request)
{
var body = UnsafeByteOperations.UnsafeWrap(new byte[request.ResponseSize]);
var payload = new Payload {
Body = body
};
return(new SimpleResponse {
Payload = payload
});
}
public static SimpleResponse CreateResponse(SimpleRequest request)
{
var data = request.ResponseSize == 0 ? Array.Empty <byte>() : new byte[request.ResponseSize];
var body = UnsafeByteOperations.UnsafeWrap(data);
var payload = new Payload {
Body = body
};
return(new SimpleResponse {
Payload = payload
});
}
public async Task CanSerializeErrors()
{
var serverCallContext = TestServerCallContextFactory.Create();
serverCallContext.UserState["__HttpContext"] = httpContextFactory();
var dispatcher = new DispatcherService();
var request = new RequestEnvelope()
{
Type = typeof(TestThrowErrorCommand).AssemblyQualifiedName,
Data = UnsafeByteOperations.UnsafeWrap(JsonSerializer.SerializeToUtf8Bytes(new TestThrowErrorCommand()))
};
var response = await dispatcher.Dispatch(request, serverCallContext);
response.ShouldNotBeNull();
}
private const int ChunkSize = 1024 * 32; // 32 KB
static async Task Main(string[] args)
{
using var channel = GrpcChannel.ForAddress("https://localhost:5001");
var client = new Uploader.UploaderClient(channel);
Console.WriteLine("Starting call");
var call = client.UploadFile();
Console.WriteLine("Sending file metadata");
await call.RequestStream.WriteAsync(new UploadFileRequest
{
Metadata = new FileMetadata
{
FileName = "pancakes.jpg"
}
});
var buffer = new byte[ChunkSize];
await using var readStream = File.OpenRead("pancakes.jpg");
while (true)
{
var count = await readStream.ReadAsync(buffer);
if (count == 0)
{
break;
}
Console.WriteLine("Sending file data chunk of length " + count);
await call.RequestStream.WriteAsync(new UploadFileRequest
{
Data = UnsafeByteOperations.UnsafeWrap(buffer.AsMemory(0, count))
});
}
Console.WriteLine("Complete request");
await call.RequestStream.CompleteAsync();
var response = await call;
Console.WriteLine("Upload id: " + response.Id);
Console.WriteLine("Shutting down");
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
public static RowOperationsPB EncodeRowOperations(List <PartialRowOperation> rows)
{
OperationsEncoder.ComputeSize(
rows,
out int rowSize,
out int indirectSize);
var rowData = new byte[rowSize];
var indirectData = new byte[indirectSize];
OperationsEncoder.Encode(rows, rowData, indirectData);
return(new RowOperationsPB
{
Rows = UnsafeByteOperations.UnsafeWrap(rowData),
IndirectData = UnsafeByteOperations.UnsafeWrap(indirectData)
});
}
public async Task Dispatch_CommandWithNoReply_EmptyResponseType()
{
var serverCallContext = TestServerCallContextFactory.Create();
serverCallContext.UserState["__HttpContext"] = httpContextFactory();
var dispatcher = new DispatcherService();
var cmd = new TestCommandNoReturnValue();
RequestEnvelope request = new RequestEnvelope()
{
Type = cmd.GetType().AssemblyQualifiedName,
Data = UnsafeByteOperations.UnsafeWrap(JsonSerializer.SerializeToUtf8Bytes(cmd))
};
var response = await dispatcher.Dispatch(request, serverCallContext);
response.Error.ShouldBeFalse();
response.Type.ShouldBeNullOrEmpty();
response.Data.IsEmpty.ShouldBeTrue();
}
public override async Task Download(DownloadRequest request, IServerStreamWriter <DownloadChunkResponse> responseStream, ServerCallContext context)
{
var filePath = Path.Combine(ToFilePath(request.FileSize));
await using var file = File.OpenRead(filePath);
while (file.Position < file.Length)
{
var array = ArrayPool <byte> .Shared.Rent(request.ChunkSize);
try
{
var bytesRead = await file.ReadAsync(array.AsMemory(), context.CancellationToken);
var chunk = UnsafeByteOperations.UnsafeWrap(array.AsMemory(0, bytesRead));
await responseStream.WriteAsync(new DownloadChunkResponse { Chunk = chunk });
}
finally
{
ArrayPool <byte> .Shared.Return(array);
}
}
}
public override async Task DownloadFile(DownloadFileRequest request, IServerStreamWriter <DownloadFileResponse> responseStream, ServerCallContext context)
{
var requestParam = request.Id;
var filename = requestParam switch
{
"4" => "pancakes4.png",
_ => "pancakes.jpg",
};
await responseStream.WriteAsync(new DownloadFileResponse
{
Metadata = new FileMetadata {
FileName = filename
}
});
var buffer = new byte[ChunkSize];
await using var fileStream = File.OpenRead(filename);
while (true)
{
var numBytesRead = await fileStream.ReadAsync(buffer);
if (numBytesRead == 0)
{
break;
}
_logger.LogInformation("Sending data chunk of {numBytesRead} bytes", numBytesRead);
await responseStream.WriteAsync(new DownloadFileResponse
{
Data = UnsafeByteOperations.UnsafeWrap(buffer.AsMemory(0, numBytesRead))
});
}
}
public ByteString UnsafeWrap()
{
return(UnsafeByteOperations.UnsafeWrap(byteBuffer));
}
private ScanRequest GetOpenRequest()
{
var request = new ScanRequestPB();
var newRequest = request.NewScanRequest = new NewScanRequestPB
{
Limit = (ulong)(_limit - _numRowsReturned),
OrderMode = _orderMode,
CacheBlocks = _cacheBlocks,
ReadMode = (Protobuf.ReadMode)_readMode,
RowFormatFlags = (ulong)RowFormatFlags.ColumnarLayout
};
newRequest.ProjectedColumns.AddRange(_columns);
// For READ_YOUR_WRITES scan, use the propagated timestamp from
// the scanner.
if (_readMode == ReadMode.ReadYourWrites)
{
long timestamp = _lowerBoundPropagationTimestamp;
if (timestamp != KuduClient.NoTimestamp)
{
newRequest.PropagatedTimestamp = (ulong)timestamp;
}
}
// If the mode is set to read on snapshot set the snapshot timestamps.
if (_readMode == ReadMode.ReadAtSnapshot)
{
if (SnapshotTimestamp != KuduClient.NoTimestamp)
{
newRequest.SnapTimestamp = (ulong)SnapshotTimestamp;
}
if (_startTimestamp != KuduClient.NoTimestamp)
{
newRequest.SnapStartTimestamp = (ulong)_startTimestamp;
}
}
if (_isFaultTolerant && _lastPrimaryKey.Length > 0)
{
newRequest.LastPrimaryKey = _lastPrimaryKey;
}
if (_startPrimaryKey.Length > 0)
{
newRequest.StartPrimaryKey = UnsafeByteOperations.UnsafeWrap(_startPrimaryKey);
}
if (_endPrimaryKey.Length > 0)
{
newRequest.StopPrimaryKey = UnsafeByteOperations.UnsafeWrap(_endPrimaryKey);
}
foreach (KuduPredicate predicate in _predicates.Values)
{
newRequest.ColumnPredicates.Add(predicate.ToProtobuf());
}
request.BatchSizeBytes = (uint)_batchSizeBytes;
return(new ScanRequest(
ScanRequestState.Opening,
request,
_schema,
_replicaSelection,
_table.TableId,
Tablet,
_partitionPruner.NextPartitionKey,
_isFaultTolerant));
}
public override async Task CreateExport(CreateExportRequest options, IServerStreamWriter <CreateExportResponse> responseStream, ServerCallContext context)
{
var ef = options.ExportFormat;
var exportFormat = (DiscordChatExporter.Core.Exporting.ExportFormat)ef;
var parsed = Snowflake.TryParse(options.ChannelId);
var channelId = parsed ?? Snowflake.Zero;
var client = new DiscordClient(options.Token);
client._tokenKind = TokenKind.Bot;
Channel channel;
try
{
channel = await client.GetChannelAsync(channelId);
}
catch (DiscordChatExporterException e)
{
if (e.Message.Contains("Authentication"))
{
throw new RpcException(new Status(StatusCode.PermissionDenied, "An invalid Discord token was provided."));
}
if (e.Message.Contains("Requested resource does not exist"))
{
throw new RpcException(new Status(StatusCode.NotFound, "A channel with the provided ID was not found."));
}
throw new RpcException(new Status(StatusCode.Unknown, $"An unknown error occurred: {e.Message}"));
}
var guild = await client.GetGuildAsync(channel.GuildId);
var res = await client.GetJsonResponseAsync("users/@me");
var me = DiscordChatExporter.Core.Discord.Data.User.Parse(res);
var path = GetPath(channel.Id.ToString(), exportFormat);
_logger.LogInformation($"[{me.FullName} ({me.Id})] Exporting #{channel.Name} ({channel.Id}) within {guild.Name} ({guild.Id}) to {path}");
var request = new ExportRequest(
guild,
channel,
path,
exportFormat,
Snowflake.TryParse(options.After),
Snowflake.TryParse(options.Before),
PartitionLimit.Null,
MessageFilter.Null,
false,
false,
options.DateFormat
);
var exporter = new ChannelExporter(client);
_logger.LogInformation("Starting export");
var progress = new Progress <double>(p => responseStream.WriteAsync(new CreateExportResponse {
Progress = p
}));
var messageCount = await exporter.ExportChannelAsync(request, progress);
_logger.LogInformation("Finished exporting");
var buffer = new byte[ChunkSize];
await using var readStream = File.OpenRead(path);
while (true)
{
var count = await readStream.ReadAsync(buffer);
if (count == 0)
{
break;
}
Console.WriteLine("Sending file data chunk of length " + count);
await responseStream.WriteAsync(new CreateExportResponse
{
Data = new ExportComplete {
MessageCount = messageCount,
Data = UnsafeByteOperations.UnsafeWrap(buffer.AsMemory(0, count))
}
});
}
deleteFile(path);
}
public static ByteString CreateFromByteArrayOptimized(byte[] array)
{
return(UnsafeByteOperations.UnsafeWrap(array));
}
public static ByteString UnsafeCreateFromBytes(byte[] buffer)
{
return(UnsafeByteOperations.UnsafeWrap(buffer));
}
public override async Task <ReplyEnvelope> Dispatch(RequestEnvelope request, ServerCallContext context)
{
var serviceProvider = context.GetHttpContext().RequestServices;
var serviceRegistry = serviceProvider.GetRequiredService <MessageHandlerRegistry>();
var logger = serviceProvider.GetRequiredService <ILogger <DispatcherService> >();
var sw = Stopwatch.StartNew();
try
{
var requestType = System.Type.GetType(request.Type, an => Assembly.Load(an.Name ?? null !), null, true, true) ?? null !;
var messageHandler = serviceRegistry.Resolve(requestType);
if (messageHandler == null)
{
throw new InvalidOperationException($"Message handler for {requestType} not found");
}
var handlerInstance = serviceProvider.GetRequiredService(messageHandler.DeclaringType ?? null !);
using var ms = new MemoryStream(request.Data.ToByteArray());
var requestMessage = await JsonSerializer.DeserializeAsync(ms, requestType) ?? null !;
var replyMessage = await messageHandler.InvokeAsync(handlerInstance, new object[] { requestMessage });
var replyType = replyMessage?.GetType();
var reply = new ReplyEnvelope
{
CorrelationId = request.CorrelationId,
Type = replyType?.AssemblyQualifiedName ?? string.Empty,
Data = replyMessage == null
? ByteString.Empty
: UnsafeByteOperations.UnsafeWrap(JsonSerializer.SerializeToUtf8Bytes(replyMessage)),
Empty = replyMessage == null
};
sw.Stop();
logger.LogInformation("GRPC Dispatch request {requestId} {requestType} responded {status} with {replyType} in {elapsed} ms", request.CorrelationId, requestType.FullName, "OK", replyType?.FullName, sw.Elapsed.TotalMilliseconds);
return(reply);
}
catch (Exception e)
{
//var serializer = new System.Runtime.Serialization.DataContractSerializer(typeof(EssApplicationException));
//using var ms = new MemoryStream();
//var writer = XmlDictionaryWriter.CreateTextWriter(ms);
//serializer.WriteObject(writer, e);
var reply = new ReplyEnvelope
{
CorrelationId = request.CorrelationId,
Error = true,
ErrorType = e.GetType().AssemblyQualifiedName,
ErrorMessage = e.Message,
ErrorDetails = e.ToString(),
//Data = UnsafeByteOperations.UnsafeWrap(ms.ToArray())
};
sw.Stop();
logger.LogError(e, "GRPC Dispatch request {requestId} {requestType} responded {status} in {elapsed} ms", request.CorrelationId, request.Type, "ERROR", sw.Elapsed.TotalMilliseconds);
return(reply);
}
}
public override ByteString?Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
// TODO - handle base64 strings without padding
return(UnsafeByteOperations.UnsafeWrap(reader.GetBytesFromBase64()));
}
private static async ValueTask <IMessage> ReadHttpBodyAsync(JsonTranscodingServerCallContext serverCallContext)
{
var httpBody = (IMessage)Activator.CreateInstance(serverCallContext.DescriptorInfo.BodyDescriptor !.ClrType) !;
var contentType = serverCallContext.HttpContext.Request.ContentType;
if (contentType != null)
{
httpBody.Descriptor.Fields[HttpBody.ContentTypeFieldNumber].Accessor.SetValue(httpBody, contentType);
}
var data = await ReadDataAsync(serverCallContext);
httpBody.Descriptor.Fields[HttpBody.DataFieldNumber].Accessor.SetValue(httpBody, UnsafeByteOperations.UnsafeWrap(data));
return(httpBody);
}
public async ValueTask <List <KuduScanToken> > BuildAsync(
CancellationToken cancellationToken = default)
{
if (LowerBoundPartitionKey.Length != 0 ||
UpperBoundPartitionKey.Length != 0)
{
throw new ArgumentException(
"Partition key bounds may not be set on KuduScanTokenBuilder");
}
// If the scan is short-circuitable, then return no tokens.
foreach (var predicate in Predicates.Values)
{
if (predicate.Type == PredicateType.None)
{
return(new List <KuduScanToken>());
}
}
var proto = new ScanTokenPB();
if (_includeTableMetadata)
{
// Set the table metadata so that a call to the master is not needed when
// deserializing the token into a scanner.
var tableMetadataPb = new TableMetadataPB
{
TableId = Table.TableId,
TableName = Table.TableName,
NumReplicas = Table.NumReplicas,
Schema = Table.SchemaPb.Schema,
PartitionSchema = Table.SchemaPb.PartitionSchema
};
if (Table.Owner is not null)
{
tableMetadataPb.Owner = Table.Owner;
}
if (Table.Comment is not null)
{
tableMetadataPb.Comment = Table.Comment;
}
if (Table.ExtraConfig.Count > 0)
{
foreach (var kvp in Table.ExtraConfig)
{
tableMetadataPb.ExtraConfigs.Add(kvp.Key, kvp.Value);
}
}
proto.TableMetadata = tableMetadataPb;
// Only include the authz token if the table metadata is included.
// It is returned in the required GetTableSchema request otherwise.
var authzToken = Client.GetAuthzToken(Table.TableId);
if (authzToken is not null)
{
proto.AuthzToken = authzToken;
}
}
else
{
// If we add the table metadata, we don't need to set the old table id
// and table name. It is expected that the creation and use of a scan token
// will be on the same or compatible versions.
proto.TableId = Table.TableId;
proto.TableName = Table.TableName;
}
// Map the column names or indices to actual columns in the table schema.
// If the user did not set either projection, then scan all columns.
var schema = Table.Schema;
if (_includeTableMetadata)
{
// If the table metadata is included, then the column indexes can be
// used instead of duplicating the ColumnSchemaPBs in the serialized
// scan token.
if (ProjectedColumnNames is not null)
{
proto.ProjectedColumnIdx.Capacity = ProjectedColumnNames.Count;
foreach (var columnName in ProjectedColumnNames)
{
var columnIndex = schema.GetColumnIndex(columnName);
proto.ProjectedColumnIdx.Add(columnIndex);
}
}
else if (ProjectedColumnIndexes is not null)
{
proto.ProjectedColumnIdx.AddRange(ProjectedColumnIndexes);
}
else
{
var numColumns = schema.Columns.Count;
proto.ProjectedColumnIdx.Capacity = numColumns;
for (int i = 0; i < numColumns; i++)
{
proto.ProjectedColumnIdx.Add(i);
}
}
}
else
{
if (ProjectedColumnNames is not null)
{
proto.ProjectedColumns.Capacity = ProjectedColumnNames.Count;
foreach (var columnName in ProjectedColumnNames)
{
int columnIndex = schema.GetColumnIndex(columnName);
var columnSchema = Table.SchemaPb.Schema.Columns[columnIndex];
proto.ProjectedColumns.Add(columnSchema);
}
}
else if (ProjectedColumnIndexes is not null)
{
proto.ProjectedColumns.Capacity = ProjectedColumnIndexes.Count;
foreach (var columnIndex in ProjectedColumnIndexes)
{
var columnSchema = Table.SchemaPb.Schema.Columns[columnIndex];
proto.ProjectedColumns.Add(columnSchema);
}
}
else
{
proto.ProjectedColumns.AddRange(Table.SchemaPb.Schema.Columns);
}
}
foreach (var predicate in Predicates.Values)
{
proto.ColumnPredicates.Add(predicate.ToProtobuf());
}
if (LowerBoundPrimaryKey.Length > 0)
{
proto.LowerBoundPrimaryKey = UnsafeByteOperations.UnsafeWrap(LowerBoundPrimaryKey);
}
if (UpperBoundPrimaryKey.Length > 0)
{
proto.UpperBoundPrimaryKey = UnsafeByteOperations.UnsafeWrap(UpperBoundPrimaryKey);
}
proto.Limit = (ulong)Limit;
proto.ReadMode = (Protobuf.ReadMode)ReadMode;
proto.ReplicaSelection = (Protobuf.ReplicaSelection)ReplicaSelection;
// If the last propagated timestamp is set send it with the scan.
long lastPropagatedTimestamp = Client.LastPropagatedTimestamp;
if (lastPropagatedTimestamp != KuduClient.NoTimestamp)
{
proto.PropagatedTimestamp = (ulong)lastPropagatedTimestamp;
}
// If the mode is set to read on snapshot set the snapshot timestamps.
if (ReadMode == ReadMode.ReadAtSnapshot)
{
if (HtTimestamp != KuduClient.NoTimestamp)
{
proto.SnapTimestamp = (ulong)HtTimestamp;
}
if (StartTimestamp != KuduClient.NoTimestamp)
{
proto.SnapStartTimestamp = (ulong)StartTimestamp;
}
}
proto.CacheBlocks = CacheBlocks;
proto.FaultTolerant = IsFaultTolerant;
proto.BatchSizeBytes = (uint)BatchSizeBytes;
// TODO:
//proto.setScanRequestTimeoutMs(scanRequestTimeout);
//proto.setKeepAlivePeriodMs(keepAlivePeriodMs);
//proto.ScanRequestTimeoutMs = 30000;
//proto.KeepAlivePeriodMs = 15000;
var pruner = PartitionPruner.Create(
schema,
Table.PartitionSchema,
Predicates,
LowerBoundPrimaryKey,
UpperBoundPrimaryKey,
LowerBoundPartitionKey,
UpperBoundPartitionKey);
var keyRanges = new List <KeyRange>();
while (pruner.HasMorePartitionKeyRanges)
{
var partitionRange = pruner.NextPartitionKeyRange;
var newKeyRanges = await Client.GetTableKeyRangesAsync(
Table.TableId,
LowerBoundPrimaryKey,
UpperBoundPrimaryKey,
partitionRange.Lower.Length == 0?null : partitionRange.Lower,
partitionRange.Upper.Length == 0?null : partitionRange.Upper,
_fetchTabletsPerRangeLookup,
_splitSizeBytes,
cancellationToken).ConfigureAwait(false);
if (newKeyRanges.Count == 0)
{
pruner.RemovePartitionKeyRange(partitionRange.Upper);
}
else
{
pruner.RemovePartitionKeyRange(
newKeyRanges[newKeyRanges.Count - 1].PartitionKeyEnd);
}
keyRanges.AddRange(newKeyRanges);
}
var tokens = new List <KuduScanToken>(keyRanges.Count);
var nowMillis = _systemClock.CurrentMilliseconds;
foreach (var keyRange in keyRanges)
{
var token = proto.Clone();
token.LowerBoundPartitionKey = UnsafeByteOperations.UnsafeWrap(keyRange.PartitionKeyStart);
token.UpperBoundPartitionKey = UnsafeByteOperations.UnsafeWrap(keyRange.PartitionKeyEnd);
var primaryKeyStart = keyRange.PrimaryKeyStart;
if (primaryKeyStart.Length > 0)
{
token.LowerBoundPrimaryKey = UnsafeByteOperations.UnsafeWrap(primaryKeyStart);
}
var primaryKeyEnd = keyRange.PrimaryKeyEnd;
if (primaryKeyEnd.Length > 0)
{
token.UpperBoundPrimaryKey = UnsafeByteOperations.UnsafeWrap(primaryKeyEnd);
}
var tablet = keyRange.Tablet;
// Set the tablet metadata so that a call to the master is not needed to
// locate the tablet to scan when opening the scanner.
if (_includeTabletMetadata)
{
// TODO: It would be more efficient to pass the TTL in instead of
// looking it up again here.
var entry = Client.GetTableLocationEntry(
Table.TableId, tablet.Partition.PartitionKeyStart);
long ttl;
if (entry is not null &&
entry.IsCoveredRange &&
(ttl = entry.Expiration - nowMillis) > 0)
{
// Build the list of server and replica metadata.
var tabletServers = tablet.Servers;
var replicas = tablet.Replicas;
var numTabletServers = tabletServers.Count;
var tabletMetadataPb = new TabletMetadataPB
{
TabletId = tablet.TabletId,
Partition = ProtobufHelper.ToPartitionPb(tablet.Partition),
TtlMillis = (ulong)ttl
};
tabletMetadataPb.TabletServers.Capacity = numTabletServers;
tabletMetadataPb.Replicas.Capacity = numTabletServers;
for (int i = 0; i < numTabletServers; i++)
{
var serverInfo = tabletServers[i];
var replica = replicas[i];
var serverMetadataPb = ProtobufHelper.ToServerMetadataPb(serverInfo);
tabletMetadataPb.TabletServers.Add(serverMetadataPb);
var replicaMetadataPb = new ReplicaMetadataPB
{
TsIdx = (uint)i,
Role = (RaftPeerPB.Types.Role)replica.Role
};
if (replica.DimensionLabel is not null)
{
replicaMetadataPb.DimensionLabel = replica.DimensionLabel;
}
tabletMetadataPb.Replicas.Add(replicaMetadataPb);
}
token.TabletMetadata = tabletMetadataPb;
}
}
tokens.Add(new KuduScanToken(keyRange, token));
}
return(tokens);
}
private static Payload CreateZerosPayload(int size)
{
return(new Payload {
Body = UnsafeByteOperations.UnsafeWrap(new byte[size])
});
}
/// <inheritdoc/>
protected override void WriteValue(ref WriteContext context, byte[] value)
{
context.WriteBytes(UnsafeByteOperations.UnsafeWrap(value));
}
