public static IEnumerable <ResultFile> Generate(IReadOnlyList <FileDescriptorProto> descriptorProtos, IEnumerable <string> filesToGenerate, IClock clock,
string grpcServiceConfigPath, IEnumerable <string> commonResourcesConfigPaths, bool generateMetadata)
{
var descriptors = FileDescriptor.BuildFromByteStrings(descriptorProtos.Select(proto => proto.ToByteString()), s_registry);
// Load side-loaded configurations; both optional.
var grpcServiceConfig = grpcServiceConfigPath != null?ServiceConfig.Parser.ParseJson(File.ReadAllText(grpcServiceConfigPath)) : null;
var commonResourcesConfigs = commonResourcesConfigPaths != null?
commonResourcesConfigPaths.Select(path => CommonResources.Parser.ParseJson(File.ReadAllText(path))) : null;
// TODO: Multi-package support not tested.
var filesToGenerateSet = filesToGenerate.ToHashSet();
var byPackage = descriptors.Where(x => filesToGenerateSet.Contains(x.Name)).GroupBy(x => x.Package).ToList();
if (byPackage.Count == 0)
{
throw new InvalidOperationException("No packages specified to build.");
}
foreach (var singlePackageFileDescs in byPackage)
{
var namespaces = singlePackageFileDescs.Select(x => x.CSharpNamespace()).Distinct().ToList();
if (namespaces.Count > 1)
{
throw new InvalidOperationException(
"All files in the same package must have the same C# namespace. " +
$"Found namespaces '{string.Join(", ", namespaces)}' in package '{singlePackageFileDescs.Key}'.");
}
var catalog = new ProtoCatalog(singlePackageFileDescs.Key, descriptors, commonResourcesConfigs);
foreach (var resultFile in GeneratePackage(namespaces[0], singlePackageFileDescs, catalog, clock, grpcServiceConfig, generateMetadata))
{
yield return(resultFile);
}
}
}
private static IReadOnlyList <FileDescriptor> GetFileDescriptors(byte[] bytes)
{
// TODO: Remove this when equivalent is in Protobuf.
// Manually read repeated field of `FileDescriptorProto` messages.
int i = 0;
var bss = new List <ByteString>();
while (i < bytes.Length)
{
if (bytes[i] != 0xa)
{
throw new InvalidOperationException($"Expected 0xa at offset {i}");
}
i += 1;
int len = 0;
int shift = 0;
while (true)
{
var b = bytes[i];
i += 1;
len |= (b & 0x7f) << shift;
shift += 7;
if ((b & 0x80) == 0)
{
break;
}
}
bss.Add(ByteString.CopyFrom(bytes, i, len));
i += len;
}
return(FileDescriptor.BuildFromByteStrings(bss));
}
private IReadOnlyCollection <FileDescriptor> ReadCompiledProtobufModel2(string pathToCompileProtobufModel)
{
using (var stream = File.OpenRead(pathToCompileProtobufModel))
{
FileDescriptorSet descriptorSet = FileDescriptorSet.Parser.ParseFrom(stream);
var byteStrings = descriptorSet.File.Select(f => f.ToByteString()).ToList();
var descriptors = FileDescriptor.BuildFromByteStrings(byteStrings);
return(descriptors);
}
}
private static FileDescriptor LoadProtos()
{
var type = typeof(DescriptorDeclarationTest);
// TODO: Make this simpler :)
FileDescriptorSet descriptorSet;
using (var stream = type.GetTypeInfo().Assembly.GetManifestResourceStream($"Google.Protobuf.Test.testprotos.pb"))
{
descriptorSet = FileDescriptorSet.Parser.ParseFrom(stream);
}
var byteStrings = descriptorSet.File.Select(f => f.ToByteString()).ToList();
var descriptors = FileDescriptor.BuildFromByteStrings(byteStrings);
return(descriptors.Single(d => d.Name == "unittest_proto3.proto"));
}
/// <summary>
/// Loads all the API models in the given descriptor set, returning them ordered by ID.
/// </summary>
public static IReadOnlyList <ApiModel> Load(FileDescriptorSet descriptorSet, string googleApisRoot)
{
var descriptorByteStrings = descriptorSet.File.Select(proto => proto.ToByteString());
var allDescriptors = FileDescriptor.BuildFromByteStrings(descriptorByteStrings, s_registry)
.ToList()
.AsReadOnly();
var models = new List <ApiModel>();
AddServiceConfigsRecursively(googleApisRoot);
return(models.OrderBy(model => model.Id, StringComparer.Ordinal).ToList().AsReadOnly());
void AddServiceConfigsRecursively(string directory)
{
// Only load service config files from versioned directories (e.g. "google/spanner/v1")
if (ApiMajorVersionPattern.IsMatch(Path.GetFileName(directory)))
{
string relativeDirectory = Path.GetRelativePath(googleApisRoot, directory).Replace('\\', '/');
var configsInDirectory = System.IO.Directory.GetFiles(directory, "*.yaml")
.Select(ServiceConfig.TryLoadFile)
.Where(config => config != null)
.ToList();
switch (configsInDirectory.Count)
{
case 0: break;
case 1:
models.Add(new ApiModel(relativeDirectory, configsInDirectory[0], allDescriptors));
break;
default:
models.Add(new ApiModel(relativeDirectory, configsInDirectory[0], allDescriptors));
Console.WriteLine($"Directory '{relativeDirectory}' contains {configsInDirectory.Count} service (API) config YAML files.");
break;
}
}
else
{
// Note: we assume there are no "nested" APIs
foreach (var subdir in System.IO.Directory.GetDirectories(directory))
{
AddServiceConfigsRecursively(subdir);
}
}
}
}
public bool Build()
{
try
{
IReadOnlyList<FileDescriptor> files;
if (m_SerializedData != null)
{
files = FileDescriptor.BuildFromByteStrings(new[] {m_SerializedData});
}
else if (m_SerializedBytes != null)
{
var fileDescriptorSet = FileDescriptorSet.Parser.ParseFrom(m_SerializedBytes);
files = FileDescriptor.BuildFromByteStrings(fileDescriptorSet.File);
}
else
{
throw new InvalidOperationException("DevDescriptor is not properly initialized.");
}
fileDescriptor = files[0];
fidelityMessage = fileDescriptor.MessageTypes.First(x => x.Name == "FidelityParams");
annotationMessage = fileDescriptor.MessageTypes.First(x => x.Name == "Annotation");
var loadingField = annotationMessage.Fields
.InDeclarationOrder()
.FirstOrDefault(DefaultMessages.IsLoadingStateField);
var sceneField = annotationMessage.Fields
.InDeclarationOrder()
.FirstOrDefault(DefaultMessages.IsSceneField);
// As message field's indexes start from 1, return 0 if field is not found.
loadingAnnotationIndex = loadingField?.FieldNumber ?? 0;
sceneAnnotationIndex = sceneField?.FieldNumber ?? 0;
annotationEnumSizes =
annotationMessage.Fields.InDeclarationOrder()
.Select(x => x.EnumType.Values.Count).ToList();
fidelityFieldNames = fidelityMessage.Fields.InDeclarationOrder().Select(x => x.Name).ToList();
return true;
}
catch (Exception e)
{
Debug.Log("Could not build DevDescriptor, " + e);
return false;
}
}
public void TestDescriptor()
{
var descriptorBytes = Hash.Descriptor.File.SerializedData;
var file = FileDescriptor.BuildFromByteStrings(new ByteString[] { descriptorBytes });
var reg = TypeRegistry.FromFiles(file);
var messageDescriptor = reg.Find("aelf.Hash");
messageDescriptor.Name.ShouldBe("Hash");
var hash = Hash.FromString("hello");
var json = JsonFormatter.Default.Format(hash);
//JsonParser.Default.Parse(json, messageDescriptor);
//it will not work, messageDescriptor clr type is null, the factory of parser is null
var deserializedHash = (Hash)JsonParser.Default.Parse(json, Hash.Descriptor);
deserializedHash.ShouldBe(hash);
}
public override async Task <int> ExecuteAsync(CommandContext context, ListSettings settings)
{
var address = settings.Address;
if (!address.StartsWith("http://") &&
!address.StartsWith("https://"))
{
address = $"https://{address}";
}
if (address.StartsWith("http://"))
{
// This switch must be set before creating the GrpcChannel/HttpClient.
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
}
var server = address;
var channel = Grpc.Net.Client.GrpcChannel.ForAddress(server);
var client = new ServerReflection.ServerReflectionClient(channel);
var stream = client.ServerReflectionInfo();
if (string.IsNullOrEmpty(settings.Service))
{
await stream.RequestStream.WriteAsync(new ServerReflectionRequest { ListServices = "ls" });
await stream.ResponseStream.MoveNext(CancellationToken.None);
foreach (var service in stream.ResponseStream.Current.ListServicesResponse.Service)
{
Console.WriteLine(service.Name);
}
await stream.RequestStream.CompleteAsync();
}
else
{
await stream.RequestStream.WriteAsync(new ServerReflectionRequest { FileContainingSymbol = settings.Service });
await stream.ResponseStream.MoveNext(CancellationToken.None);
var descriptors = FileDescriptor.BuildFromByteStrings(stream.ResponseStream.Current.FileDescriptorResponse.FileDescriptorProto);
await stream.RequestStream.CompleteAsync();
var service = descriptors
.SelectMany(x => x.Services)
.FirstOrDefault(x => string.Equals(x.FullName, settings.Service));
if (service is object)
{
Console.WriteLine($"filename: {service.File.Name}");
Console.WriteLine($"package: {service.File.Package}");
Console.WriteLine($"service {service.Name} {{");
foreach (var method in service.Methods)
{
Console.WriteLine($" {GetMethod(method)}");
}
Console.WriteLine("}");
}
else
{
var method = descriptors
.SelectMany(x => x.Services)
.SelectMany(x => x.Methods)
.FirstOrDefault(x => string.Equals(x.FullName, settings.Service));
Console.WriteLine(GetMethod(method));
}
public override async Task <int> ExecuteAsync(CommandContext context, DumpSettings settings)
{
var address = settings.Address;
if (!address.StartsWith("http://") &&
!address.StartsWith("https://"))
{
address = $"https://{address}";
}
if (address.StartsWith("http://"))
{
// This switch must be set before creating the GrpcChannel/HttpClient.
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
}
var server = address;
var channel = Grpc.Net.Client.GrpcChannel.ForAddress(server);
var client = new ServerReflection.ServerReflectionClient(channel);
var stream = client.ServerReflectionInfo();
await stream.RequestStream.WriteAsync(new ServerReflectionRequest { FileContainingSymbol = settings.Service });
await stream.ResponseStream.MoveNext(CancellationToken.None);
var descriptors = FileDescriptor.BuildFromByteStrings(stream.ResponseStream.Current.FileDescriptorResponse.FileDescriptorProto);
await stream.RequestStream.CompleteAsync();
// Output
if (!string.IsNullOrWhiteSpace(settings.Output))
{
settings.Output = Path.GetFullPath(settings.Output);
if (!Directory.Exists(settings.Output))
{
Directory.CreateDirectory(settings.Output);
}
}
foreach (var descriptor in descriptors)
{
if (IsWellKnownType(descriptor))
{
continue;
}
TextWriter writer;
if (string.IsNullOrWhiteSpace(settings.Output))
{
writer = Console.Out;
await writer.WriteLineAsync("---");
await writer.WriteAsync("File: ");
await writer.WriteLineAsync(descriptor.Name);
await writer.WriteLineAsync("---");
}
else
{
var path = Path.Join(settings.Output, descriptor.Name);
var directory = Path.GetDirectoryName(path);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
writer = File.CreateText(path.Replace("/", "\\"));
}
await WriteFileDescriptor(descriptor, writer);
await writer.DisposeAsync();
}
return(0);
}
public static IEnumerable <ResultFile> Generate(IReadOnlyList <FileDescriptorProto> descriptorProtos, IEnumerable <string> filesToGenerate, IClock clock,
string grpcServiceConfigPath, string serviceConfigPath, IEnumerable <string> commonResourcesConfigPaths, ApiTransports transports)
{
var descriptors = FileDescriptor.BuildFromByteStrings(descriptorProtos.Select(proto => proto.ToByteString()), s_registry);
// Load side-loaded configurations; both optional.
var grpcServiceConfig = grpcServiceConfigPath is object?ServiceConfig.Parser.ParseJson(File.ReadAllText(grpcServiceConfigPath)) : null;
var serviceConfig = ParseServiceConfigYaml(serviceConfigPath);
var commonResourcesConfigs = commonResourcesConfigPaths != null?
commonResourcesConfigPaths.Select(path => CommonResources.Parser.ParseJson(File.ReadAllText(path))) : null;
// TODO: Multi-package support not tested.
var filesToGenerateSet = filesToGenerate.ToHashSet();
var byPackage = descriptors.Where(x => filesToGenerateSet.Contains(x.Name)).GroupBy(x => x.Package).ToList();
if (byPackage.Count == 0)
{
throw new InvalidOperationException("No packages specified to build.");
}
// Collect all service details here to emit one `gapic_metadata.json` file for multi-package situations (e.g. Kms with Iam)
var allServiceDetails = new List <ServiceDetails>();
foreach (var singlePackageFileDescs in byPackage)
{
var namespaces = singlePackageFileDescs.Select(x => x.CSharpNamespace()).Distinct().ToList();
if (namespaces.Count > 1)
{
throw new InvalidOperationException(
"All files in the same package must have the same C# namespace. " +
$"Found namespaces '{string.Join(", ", namespaces)}' in package '{singlePackageFileDescs.Key}'.");
}
var catalog = new ProtoCatalog(singlePackageFileDescs.Key, descriptors, singlePackageFileDescs, commonResourcesConfigs);
foreach (var resultFile in GeneratePackage(namespaces[0], singlePackageFileDescs, catalog, clock, grpcServiceConfig, serviceConfig, allServiceDetails, transports))
{
yield return(resultFile);
}
}
// We assume that the first service we've generated corresponds to the service config (if we have one),
// and is a service from the primary library we're generating. This is used for API validation and
// gapic_metadata.json generation. This means it doesn't matter (for gapic_metadata.json)
// if we're actually asked to generate more services than we really want. This currently
// happens for services with IAM/location mix-ins, for example.
var primaryLibraryProtoPackage = allServiceDetails.FirstOrDefault()?.ProtoPackage;
var primaryLibraryServices = allServiceDetails.Where(s => s.ProtoPackage == primaryLibraryProtoPackage).ToList();
if (primaryLibraryServices.Any())
{
// Generate gapic_metadata.json, if there are any services.
var gapicMetadataJsonContent = MetadataGenerator.GenerateGapicMetadataJson(primaryLibraryServices);
yield return(new ResultFile("gapic_metadata.json", gapicMetadataJsonContent));
}
var unhandledApis = (serviceConfig?.Apis.Select(api => api.Name) ?? Enumerable.Empty <string>())
.Except(primaryLibraryServices.Select(s => s.ServiceFullName))
.Except(AllowedAdditionalServices)
.ToList();
if (unhandledApis.Any())
{
throw new InvalidOperationException($"Unhandled APIs in service config: {string.Join(", ", unhandledApis)}");
}
}
