/// <summary>
/// Constructs a new instance for a given service.
/// </summary>
/// <param name="serviceDescriptor">The protobuf descriptor for the service.</param>
/// <param name="defaultEndpoint">The default endpoint to connect to.</param>
/// <param name="defaultScopes">The default scopes for the service. Must not be null, and must not contain any null elements. May be empty.</param>
/// <param name="supportsScopedJwts">Whether the service supports scoped JWTs as credentials.</param>
/// <param name="transports">The transports supported by this service.</param>
/// <param name="apiMetadata">The metadata for this API, including all of the services expected to be available at the same endpoint, and all associated protos.</param>
public ServiceMetadata(ServiceDescriptor serviceDescriptor, string defaultEndpoint, IEnumerable <string> defaultScopes, bool supportsScopedJwts, ApiTransports transports, ApiMetadata apiMetadata)
{
ServiceDescriptor = GaxPreconditions.CheckNotNull(serviceDescriptor, nameof(serviceDescriptor));
Name = serviceDescriptor.Name;
GaxPreconditions.CheckArgument(Name.Length > 0, nameof(serviceDescriptor), "Service has an empty name");
DefaultEndpoint = defaultEndpoint;
DefaultScopes = GaxPreconditions.CheckNotNull(defaultScopes, nameof(defaultScopes)).ToList().AsReadOnly();
GaxPreconditions.CheckArgument(!DefaultScopes.Any(x => x == null), nameof(defaultScopes), "Scopes must not contain any null references");
SupportsScopedJwts = supportsScopedJwts;
Transports = transports;
ApiMetadata = GaxPreconditions.CheckNotNull(apiMetadata, nameof(apiMetadata));
}
public ServiceDetails(ProtoCatalog catalog, string ns, ServiceDescriptor desc, ServiceConfig grpcServiceConfig, Service serviceConfig, ApiTransports transports)
{
Catalog = catalog;
Namespace = ns;
ProtoPackage = desc.File.Package;
PackageVersion = ProtoPackage.Split('.').FirstOrDefault(part => ApiVersionPattern.IsMatch(part));
DocLines = desc.Declaration.DocLines().ToList();
SnippetsNamespace = $"{ns}.Snippets";
UnitTestsNamespace = $"{ns}.Tests";
// Must come early; used by `MethodDetails.Create()`
MethodGrpcConfigsByName = grpcServiceConfig?.MethodConfig
.SelectMany(conf => conf.Name.Select(name => (name, conf)))
.Where(x => x.name.Service == desc.FullName)
.ToImmutableDictionary(x => $"{x.name.Service}/{x.name.Method}", x => x.conf) ??
ImmutableDictionary <string, MethodConfig> .Empty;
ServiceFullName = desc.FullName;
ServiceName = desc.Name;
DocumentationName = desc.Name; // TODO: There may be a more suitable name than this.
ProtoTyp = Typ.Manual(ns, desc.Name);
GrpcClientTyp = Typ.Nested(ProtoTyp, $"{desc.Name}Client");
SettingsTyp = Typ.Manual(ns, $"{desc.Name}Settings");
BuilderTyp = Typ.Manual(ns, $"{desc.Name}ClientBuilder");
ClientAbstractTyp = Typ.Manual(ns, $"{desc.Name}Client");
ClientImplTyp = Typ.Manual(ns, $"{desc.Name}ClientImpl");
DefaultHost = desc.GetExtension(ClientExtensions.DefaultHost) ?? "";
// We need to account for regional default endpoints like "us-east1-pubsub.googleapis.com"
DefaultHostServiceName = DefaultHost
.Split('.', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault()
?.Split('-', StringSplitOptions.RemoveEmptyEntries).LastOrDefault();
DefaultPort = 443; // Hardcoded; this is not specifiable by proto annotation.
var oauthScopes = desc.GetExtension(ClientExtensions.OauthScopes);
DefaultScopes = string.IsNullOrEmpty(oauthScopes) ? Enumerable.Empty <string>() : oauthScopes.Split(',', ' ');
Methods = desc.Methods.Select(x => MethodDetails.Create(this, x)).ToList();
ServiceSnippetsTyp = Typ.Manual(SnippetsNamespace, $"AllGenerated{desc.Name}ClientSnippets");
SnippetsTyp = Typ.Manual(SnippetsNamespace, $"Generated{desc.Name}ClientSnippets");
SnippetsClientName = $"{desc.Name.ToLowerCamelCase()}Client";
UnitTestsTyp = Typ.Manual(UnitTestsNamespace, $"Generated{desc.Name}ClientTest");
NonStandardLro = NonStandardLroDetails.ForService(desc);
Mixins = serviceConfig?.Apis
.Select(api => AvailableMixins.GetValueOrDefault(api.Name))
.Where(mixin => mixin is object)
// Don't use mixins within the package that contains that mixin.
.Where(mixin => mixin.GapicClientType.Namespace != ns)
.ToList() ?? Enumerable.Empty <MixinDetails>();
Transports = transports;
}
internal static ServiceMetadata WithTransports(this ServiceMetadata metadata, ApiTransports transports) => new ServiceMetadata(metadata.ServiceDescriptor, metadata.DefaultEndpoint, metadata.DefaultScopes, metadata.SupportsScopedJwts, transports, metadata.ApiMetadata);
private protected GrpcAdapter(ApiTransports supportedTransports)
{
_supportedTransports = supportedTransports;
}
private void ProtoTestSingle(string testProtoName,
bool ignoreCsProj = false, bool ignoreSnippets = false, bool ignoreUnitTests = false,
string grpcServiceConfigPath = null, string serviceConfigPath = null, IEnumerable <string> commonResourcesConfigPaths = null, ApiTransports transports = ApiTransports.Grpc,
bool ignoreGapicMetadataFile = true, bool ignoreApiMetadataFile = true, bool ignoreServiceExtensionsFile = true) =>
ProtoTestSingle(
new[] { testProtoName },
// The following three don't need to be customized for simple cases
sourceDir: null,
outputDir: null,
package: null,
ignoreCsProj,
ignoreSnippets,
ignoreUnitTests,
grpcServiceConfigPath,
serviceConfigPath,
commonResourcesConfigPaths,
transports,
ignoreGapicMetadataFile,
ignoreApiMetadataFile,
ignoreServiceExtensionsFile
);
private void ProtoTestSingle(IEnumerable <string> testProtoNames, string sourceDir = null, string outputDir = null, string package = null,
bool ignoreCsProj = false, bool ignoreSnippets = false, bool ignoreUnitTests = false,
string grpcServiceConfigPath = null, string serviceConfigPath = null, IEnumerable <string> commonResourcesConfigPaths = null, ApiTransports transports = ApiTransports.Grpc,
bool ignoreGapicMetadataFile = true, bool ignoreApiMetadataFile = true, bool ignoreServiceExtensionsFile = true)
{
// Confirm each generated file is identical to the expected output.
// Use `// TEST_START` and `// TEST_END` lines in the expected file to test subsets of output files.
// Or include `// TEST_DISABLE` to disable testing of the entire file.
sourceDir = sourceDir ?? testProtoNames.First();
outputDir = outputDir ?? sourceDir;
var protoPaths = testProtoNames.Select(x => Path.Combine("ProtoTests", sourceDir, $"{x}.proto"));
package = package ?? $"testing.{sourceDir.ToLowerInvariant()}";
var files = Run(protoPaths, package,
grpcServiceConfigPath, serviceConfigPath, commonResourcesConfigPaths, transports);
// Check output is present.
Assert.NotEmpty(files);
// Write all output files to the temporary directory before validating any.
// This makes it easier to see the complete set of outputs.
foreach (var file in files)
{
var pathToWriteTo = Path.Combine(Invoker.ActualGeneratedFilesDir, outputDir, file.RelativePath);
Directory.CreateDirectory(Path.GetDirectoryName(pathToWriteTo));
File.WriteAllText(pathToWriteTo, file.Content);
}
// Verify each output file.
foreach (var file in files)
{
if ((ignoreCsProj && file.RelativePath.EndsWith(".csproj")) ||
(ignoreSnippets && file.RelativePath.Contains($".Snippets{Path.DirectorySeparatorChar}")) ||
(ignoreSnippets && file.RelativePath.Contains($".GeneratedSnippets{Path.DirectorySeparatorChar}")) ||
(ignoreUnitTests && file.RelativePath.Contains($".Tests{Path.DirectorySeparatorChar}")) ||
(ignoreGapicMetadataFile && file.RelativePath.EndsWith("gapic_metadata.json")) ||
(ignoreApiMetadataFile && file.RelativePath.EndsWith(PackageApiMetadataGenerator.FileName)) ||
(ignoreServiceExtensionsFile && file.RelativePath.EndsWith(ServiceCollectionExtensionsGenerator.FileName)))
{
continue;
}
var expectedFilePath = Path.Combine(Invoker.GeneratorTestsDir, "ProtoTests", outputDir, file.RelativePath);
TextComparer.CompareText(expectedFilePath, file);
}
}
private IEnumerable <ResultFile> Run(IEnumerable <string> protoFilenames, string package,
string grpcServiceConfigPath, string serviceConfigPath, IEnumerable <string> commonResourcesConfigPaths, ApiTransports transports)
{
var clock = new FakeClock(new DateTime(2019, 1, 1));
var protoPaths = protoFilenames.Select(x => Path.Combine(Invoker.GeneratorTestsDir, x));
using (var desc = Invoker.TempFile())
{
var args = new[]
{
"-o", desc.Path,
"--include_imports",
"--include_source_info",
$"-I{Invoker.CommonProtosDir}",
$"-I{Invoker.ProtobufDir}",
$"-I{Invoker.GeneratorTestsDir}",
}.Concat(protoPaths);
Invoker.Protoc(string.Join(" ", args));
var descriptorBytes = File.ReadAllBytes(desc.Path);
FileDescriptorSet descriptorSet = FileDescriptorSet.Parser.ParseFrom(descriptorBytes);
return(CodeGenerator.Generate(descriptorSet, package, clock, grpcServiceConfigPath, serviceConfigPath, commonResourcesConfigPaths, transports));
}
}
public static IEnumerable <ResultFile> Generate(FileDescriptorSet descriptorSet, string package, IClock clock,
string grpcServiceConfigPath, string serviceConfigPath, IEnumerable <string> commonResourcesConfigPaths, ApiTransports transports)
{
var descriptors = descriptorSet.File;
var filesToGenerate = descriptors.Where(x => x.Package == package).Select(x => x.Name).ToList();
return(Generate(descriptors, filesToGenerate, clock, grpcServiceConfigPath, serviceConfigPath, commonResourcesConfigPaths, transports));
}
private static IEnumerable <ResultFile> GeneratePackage(string ns,
IEnumerable <FileDescriptor> packageFileDescriptors, ProtoCatalog catalog, IClock clock,
ServiceConfig grpcServiceConfig, Service serviceConfig, List <ServiceDetails> allServiceDetails, ApiTransports transports)
{
var clientPathPrefix = $"{ns}{Path.DirectorySeparatorChar}";
var serviceSnippetsPathPrefix = $"{ns}.Snippets{Path.DirectorySeparatorChar}";
var snippetsPathPrefix = $"{ns}.GeneratedSnippets{Path.DirectorySeparatorChar}";
var unitTestsPathPrefix = $"{ns}.Tests{Path.DirectorySeparatorChar}";
bool hasLro = false;
HashSet <string> mixins = new HashSet <string>();
bool hasContent = false;
var packageServiceDetails = new List <ServiceDetails>();
HashSet <string> allResourceNameClasses = new HashSet <string>();
HashSet <string> duplicateResourceNameClasses = new HashSet <string>();
IList <Snippet> snippets = new List <Snippet>();
IEnumerable <Typ> packageTyps = packageFileDescriptors.SelectMany(
fileDescriptor => fileDescriptor.Services.Select(serv => Typ.Manual(ns, serv.Name))
.Union(fileDescriptor.MessageTypes.Select(msg => Typ.Manual(ns, msg.Name)))
.Union(fileDescriptor.EnumTypes.Select(e => Typ.Manual(ns, e.Name, isEnum: true))));
var seenPaginatedResponseTyps = new HashSet <Typ>();
foreach (var fileDesc in packageFileDescriptors)
{
foreach (var service in fileDesc.Services)
{
// Generate settings and client code for requested package.
var serviceDetails = new ServiceDetails(catalog, ns, service, grpcServiceConfig, serviceConfig, transports);
packageServiceDetails.Add(serviceDetails);
var ctx = SourceFileContext.CreateFullyAliased(clock, s_wellknownNamespaceAliases);
var code = ServiceCodeGenerator.Generate(ctx, serviceDetails, seenPaginatedResponseTyps);
var filename = $"{clientPathPrefix}{serviceDetails.ClientAbstractTyp.Name}.g.cs";
yield return(new ResultFile(filename, code));
// Generate service-per-file snippets for the service
// TODO: Consider removing this once we have integrated the snippet-per-file snippets
// with docs generation.
var serviceSnippetsCtx = SourceFileContext.CreateUnaliased(
clock, s_wellknownNamespaceAliases, s_avoidAliasingNamespaceRegex, packageTyps, maySkipOwnNamespaceImport: true);
var serviceSnippetsCode = SnippetCodeGenerator.Generate(serviceSnippetsCtx, serviceDetails);
var serviceSnippetsFilename = $"{serviceSnippetsPathPrefix}{serviceDetails.ClientAbstractTyp.Name}Snippets.g.cs";
yield return(new ResultFile(serviceSnippetsFilename, serviceSnippetsCode));
// Generate snippet-per-file snippets for the service
// TODO: Integrate snippet-per-file snippets with docs generation.
foreach (var snippetGenerator in SnippetCodeGenerator.SnippetsGenerators(serviceDetails))
{
var snippetCtx = SourceFileContext.CreateUnaliased(
clock, s_wellknownNamespaceAliases, s_avoidAliasingNamespaceRegex, packageTyps, maySkipOwnNamespaceImport: false);
var(snippetCode, snippetMetadata) = snippetGenerator.Generate(snippetCtx);
snippetMetadata.File = $"{serviceDetails.ClientAbstractTyp.Name}.{snippetGenerator.SnippetMethodName}Snippet.g.cs";
var snippetFile = $"{snippetsPathPrefix}{snippetMetadata.File}";
snippets.Add(snippetMetadata);
yield return(new ResultFile(snippetFile, snippetCode));
}
// Generate unit tests for the the service.
var unitTestCtx = SourceFileContext.CreateFullyAliased(clock, s_wellknownNamespaceAliases);
var unitTestCode = UnitTestCodeGeneration.Generate(unitTestCtx, serviceDetails);
var unitTestFilename = $"{unitTestsPathPrefix}{serviceDetails.ClientAbstractTyp.Name}Test.g.cs";
yield return(new ResultFile(unitTestFilename, unitTestCode));
// Record whether LRO/mixins are used.
hasLro |= serviceDetails.Methods.Any(x => x is MethodDetails.Lro);
foreach (var mixin in serviceDetails.Mixins)
{
mixins.Add(mixin.GrpcServiceName);
}
hasContent = true;
}
var resCtx = SourceFileContext.CreateFullyAliased(clock, s_wellknownNamespaceAliases);
var(resCode, resCodeClassCount) = ResourceNamesGenerator.Generate(catalog, resCtx, fileDesc);
// Only produce an output file if it contains >0 [partial] classes.
if (resCodeClassCount > 0)
{
// Keep track of the resource names, to spot duplicates
var resourceNameClasses = catalog.GetResourceDefsByFile(fileDesc)
.Where(def => def.HasNotWildcard && !def.IsCommon)
.Select(def => def.ResourceNameTyp.Name);
foreach (var resourceNameClass in resourceNameClasses)
{
if (!allResourceNameClasses.Add(resourceNameClass))
{
duplicateResourceNameClasses.Add(resourceNameClass);
}
}
// Yield the file to be generated.
var filenamePrefix = Path.GetFileNameWithoutExtension(fileDesc.Name).ToUpperCamelCase();
var resFilename = $"{clientPathPrefix}{filenamePrefix}ResourceNames.g.cs";
yield return(new ResultFile(resFilename, resCode));
hasContent = true;
}
var lroAdaptationCtx = SourceFileContext.CreateFullyAliased(clock, s_wellknownNamespaceAliases);
var(lroCode, lroClassCount) = LroAdaptationGenerator.Generate(catalog, lroAdaptationCtx, fileDesc);
if (lroClassCount > 0)
{
var filenamePrefix = Path.GetFileNameWithoutExtension(fileDesc.Name).ToUpperCamelCase();
var lroAdaptationFilename = $"{clientPathPrefix}{filenamePrefix}LroAdaptation.g.cs";
yield return(new ResultFile(lroAdaptationFilename, lroCode));
hasContent = true;
}
}
// Now we've processed all the files, check for duplicate resource names.
if (duplicateResourceNameClasses.Count > 0)
{
throw new InvalidOperationException($"The following resource name classes were created multiple times: {string.Join(", ", duplicateResourceNameClasses)}");
}
// Only output csproj's and snippet metadata if there is any other generated content.
// When processing a (proto) package without any services there will be no generated content.
if (hasContent)
{
// Generate client csproj.
var csprojContent = CsProjGenerator.GenerateClient(hasLro, mixins);
var csprojFilename = $"{clientPathPrefix}{ns}.csproj";
yield return(new ResultFile(csprojFilename, csprojContent));
// If we only generated resources, we don't need to generate all of these.
if (packageServiceDetails.Count > 0)
{
allServiceDetails.AddRange(packageServiceDetails);
// Generate the package-wide API metadata
var ctx = SourceFileContext.CreateFullyAliased(clock, s_wellknownNamespaceAliases);
var packageApiMetadataContent = PackageApiMetadataGenerator.GeneratePackageApiMetadata(ns, ctx, packageFileDescriptors);
var packageApiMetadataFilename = $"{clientPathPrefix}{PackageApiMetadataGenerator.FileName}";
yield return(new ResultFile(packageApiMetadataFilename, packageApiMetadataContent));
// Generate snippets csproj.
var serviceSnippetsCsprojContent = CsProjGenerator.GenerateSnippets(ns);
var serviceSnippetsCsProjFilename = $"{serviceSnippetsPathPrefix}{ns}.Snippets.csproj";
yield return(new ResultFile(serviceSnippetsCsProjFilename, serviceSnippetsCsprojContent));
// Generate dependency injection extension methods
var dependencyInjectionExtensionsContent = ServiceCollectionExtensionsGenerator.GenerateExtensions(ctx, packageServiceDetails);
var dependencyInjectionExtensionsFilename = $"{clientPathPrefix}{ServiceCollectionExtensionsGenerator.FileName}";
yield return(new ResultFile(dependencyInjectionExtensionsFilename, dependencyInjectionExtensionsContent));
// Generate snippet metadata (only for snippet-per-file snippets).
// All services in this package have the same package information, namespace etc. so, we
// just pick the first one
var serviceDetails = packageServiceDetails.First();
var snippetIndexJsonContent = SnippetCodeGenerator.GenerateSnippetIndexJson(snippets, serviceDetails);
yield return(new ResultFile($"{snippetsPathPrefix}snippet_metadata_{serviceDetails.ProtoPackage}.json", snippetIndexJsonContent));
// Generate snippet-per-file snippets csproj.
var snippetsCsprojContent = CsProjGenerator.GenerateSnippets(ns);
var snippetsCsProjFilename = $"{snippetsPathPrefix}{ns}.GeneratedSnippets.csproj";
yield return(new ResultFile(snippetsCsProjFilename, snippetsCsprojContent));
// Generate unit-tests csproj.
var unitTestsCsprojContent = CsProjGenerator.GenerateUnitTests(ns);
var unitTestsCsprojFilename = $"{unitTestsPathPrefix}{ns}.Tests.csproj";
yield return(new ResultFile(unitTestsCsprojFilename, unitTestsCsprojContent));
}
}
}
public static IEnumerable <ResultFile> Generate(IReadOnlyList <FileDescriptorProto> descriptorProtos, IEnumerable <string> filesToGenerate, IClock clock,
string grpcServiceConfigPath, string serviceConfigPath, IEnumerable <string> commonResourcesConfigPaths, ApiTransports transports)
{
var descriptors = FileDescriptor.BuildFromByteStrings(descriptorProtos.Select(proto => proto.ToByteString()), s_registry);
// Load side-loaded configurations; both optional.
var grpcServiceConfig = grpcServiceConfigPath is object?ServiceConfig.Parser.ParseJson(File.ReadAllText(grpcServiceConfigPath)) : null;
var serviceConfig = ParseServiceConfigYaml(serviceConfigPath);
var commonResourcesConfigs = commonResourcesConfigPaths != null?
commonResourcesConfigPaths.Select(path => CommonResources.Parser.ParseJson(File.ReadAllText(path))) : null;
// TODO: Multi-package support not tested.
var filesToGenerateSet = filesToGenerate.ToHashSet();
var byPackage = descriptors.Where(x => filesToGenerateSet.Contains(x.Name)).GroupBy(x => x.Package).ToList();
if (byPackage.Count == 0)
{
throw new InvalidOperationException("No packages specified to build.");
}
// Collect all service details here to emit one `gapic_metadata.json` file for multi-package situations (e.g. Kms with Iam)
var allServiceDetails = new List <ServiceDetails>();
foreach (var singlePackageFileDescs in byPackage)
{
var namespaces = singlePackageFileDescs.Select(x => x.CSharpNamespace()).Distinct().ToList();
if (namespaces.Count > 1)
{
throw new InvalidOperationException(
"All files in the same package must have the same C# namespace. " +
$"Found namespaces '{string.Join(", ", namespaces)}' in package '{singlePackageFileDescs.Key}'.");
}
var catalog = new ProtoCatalog(singlePackageFileDescs.Key, descriptors, singlePackageFileDescs, commonResourcesConfigs);
foreach (var resultFile in GeneratePackage(namespaces[0], singlePackageFileDescs, catalog, clock, grpcServiceConfig, serviceConfig, allServiceDetails, transports))
{
yield return(resultFile);
}
}
// We assume that the first service we've generated corresponds to the service config (if we have one),
// and is a service from the primary library we're generating. This is used for API validation and
// gapic_metadata.json generation. This means it doesn't matter (for gapic_metadata.json)
// if we're actually asked to generate more services than we really want. This currently
// happens for services with IAM/location mix-ins, for example.
var primaryLibraryProtoPackage = allServiceDetails.FirstOrDefault()?.ProtoPackage;
var primaryLibraryServices = allServiceDetails.Where(s => s.ProtoPackage == primaryLibraryProtoPackage).ToList();
if (primaryLibraryServices.Any())
{
// Generate gapic_metadata.json, if there are any services.
var gapicMetadataJsonContent = MetadataGenerator.GenerateGapicMetadataJson(primaryLibraryServices);
yield return(new ResultFile("gapic_metadata.json", gapicMetadataJsonContent));
}
var unhandledApis = (serviceConfig?.Apis.Select(api => api.Name) ?? Enumerable.Empty <string>())
.Except(primaryLibraryServices.Select(s => s.ServiceFullName))
.Except(AllowedAdditionalServices)
.ToList();
if (unhandledApis.Any())
{
throw new InvalidOperationException($"Unhandled APIs in service config: {string.Join(", ", unhandledApis)}");
}
}
internal FakeGrpcAdapter(ApiTransports transports = ApiTransports.Grpc) : base(transports)
{
}