static void GenerateProjects(string apiRoot, ApiMetadata api, HashSet <string> apiNames)
{
// We assume the source directories already exist, either because they've just
// been generated or because they were already there. We infer the type of each
// project based on the directory name. Expected suffixes:
// - None: main API
// - .Snippets: snippets (manual and generated)
// - .Tests: unit tests
// - .IntegrationTests: integration tests
// Anything else will be ignored for now...
var projectDirectories = Directory.GetDirectories(apiRoot)
.Where(pd => Path.GetFileName(pd).StartsWith(api.Id))
.ToList();
foreach (var dir in projectDirectories)
{
string suffix = Path.GetFileName(dir).Substring(api.Id.Length);
switch (suffix)
{
case "":
GenerateMainProject(api, dir, apiNames);
break;
case ".IntegrationTests":
case ".Snippets":
case ".Tests":
GenerateTestProject(api, dir, apiNames);
GenerateCoverageFile(api, dir);
break;
}
}
// TODO: Updates for unknown project types? Tricky...
}
/// <summary>
/// Generates a metadata file (currently .repo-metadata.json; may change name later) with
/// all the information that language-agnostic tools require.
/// </summary>
public static void GenerateMetadataFile(string apiRoot, ApiMetadata api)
{
string metadataPath = Path.Combine(apiRoot, ".repo-metadata.json");
var version = api.StructuredVersion;
string versionBasedReleaseLevel =
// Version "1.0.0-beta00" hasn't been released at all, so we don't have a package to talk about.
(version.Prerelease ?? "").EndsWith("00") && version.Major == 1 && version.Minor == 0 ? "none"
// If it's not a prerelease now, or it's ever got to 1.0, it's generally "ga"
: version.Major > 1 || version.Minor > 0 || version.Prerelease == null ? "ga"
: version.Prerelease.StartsWith("beta") ? "beta" : "alpha";
string releaseLevel = api.ReleaseLevelOverride ?? versionBasedReleaseLevel;
if (releaseLevel == "none")
{
// If we have temporarily set the version to (say) beta01 and then reset it to beta00,
// make sure we don't have an obsolete metadata file.
File.Delete(metadataPath);
return;
}
var metadata = new
{
distribution_name = api.Id,
release_level = releaseLevel,
client_documentation = $"https://googleapis.dev/dotnet/{api.Id}/latest",
};
string json = JsonConvert.SerializeObject(metadata, Formatting.Indented);
File.WriteAllText(metadataPath, json);
}
private static void GenerateSynthConfiguration(string apiRoot, ApiMetadata api)
{
if (api.Generator == GeneratorType.None)
{
return;
}
var synthFile = Path.Combine(apiRoot, "synth.py");
// Currently all APIs use the exact same synth file, so we can just replace it every time.
// We may need something more sophisticated in the future.
string content =
@" # GENERATED BY Google.Cloud.Tools.ProjectGenerator - DO NOT EDIT!
import sys
from synthtool import shell
from pathlib import Path
# Parent of the script is the API-specific directory
# Parent of the API-specific directory is the apis directory
# Parent of the apis directory is the repo root
root = Path(__file__).parent.parent.parent
package = Path(__file__).parent.name
bash = '/bin/bash'
if sys.platform == 'win32':
bash = 'C:\\Program Files\\Git\\bin\\bash.exe'
shell.run(
(bash, 'generateapis.sh', '--check_compatibility', package),
cwd = root,
hide_output = False)
";
File.WriteAllText(synthFile, content);
}
private static void GenerateTestProject(ApiMetadata api, string directory)
{
var dependencies = new SortedList <string, string>(api.TestDependencies)
{
{ $@"..\{api.Id}\{api.Id}.csproj", "" }, // Main project
{ "Microsoft.NET.Test.Sdk", "15.0.0" },
{ "xunit", "2.3.0-beta1-build3642" },
{ "xunit.runner.visualstudio", "2.3.0-beta1-build1309 " },
{ "Moq", "4.7.8" }
};
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFrameworks", api.TestTargetFrameworks ?? api.TargetFrameworks ?? "netcoreapp1.0;net452"),
new XElement("Features", "IOperation"),
new XElement("IsPackable", false),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("PublicSign", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), true),
new XElement("TreatWarningsAsErrors", true),
// 1701, 1702 and 1705 are disabled by default.
// 4014 is required as snippets for streaming samples call Task.Run and don't await the result.
// See https://github.com/googleapis/toolkit/issues/1271 - when that's fixed, we can remove this.
new XElement("NoWarn", "1701;1702;1705;4014")
);
var itemGroup = CreateDependenciesElement(dependencies, api.IsReleaseVersion);
// Allow test projects to use dynamic...
itemGroup.Add(new XElement("Reference",
new XAttribute("Condition", "'$(TargetFramework)' == 'net452'"),
new XAttribute("Include", "Microsoft.CSharp")));
// Test service... it keeps on getting added by Visual Studio, so let's just include it everywhere.
itemGroup.Add(new XElement("Service", new XAttribute("Include", "{82a7f48d-3b50-4b1e-b82e-3ada8210c358}")));
WriteProjectFile(api, directory, propertyGroup, itemGroup, null);
}
static int Main()
{
try
{
ValidateCommonHiddenProductionDependencies();
var root = DirectoryLayout.DetermineRootDirectory();
var apis = ApiMetadata.LoadApis();
Console.WriteLine($"API catalog contains {apis.Count} entries");
HashSet <string> apiNames = new HashSet <string>(apis.Select(api => api.Id));
foreach (var api in apis)
{
GenerateProjects(Path.Combine(root, "apis", api.Id), api, apiNames);
}
foreach (var api in apis)
{
GenerateSolutionFiles(Path.Combine(root, "apis", api.Id), api);
}
foreach (var api in apis)
{
GenerateDocumentationStub(Path.Combine(root, "apis", api.Id), api);
}
return(0);
}
catch (UserErrorException e)
{
Console.WriteLine($"Configuration error: {e.Message}");
return(1);
}
catch (Exception e)
{
Console.WriteLine($"Failed: {e}");
return(1);
}
}
private static void GenerateCoverageFile(ApiMetadata api, string directory)
{
// Don't generate a coverage file if we've got a placeholder directory
if (Directory.GetFiles(directory, "*.cs", SearchOption.AllDirectories).Length == 0)
{
return;
}
var filters = new XElement("Filters", new XElement("IncludeFilters",
new XElement("FilterEntry", new XElement("ModuleMask", api.Id)),
// Allow tests to contribute coverage to project dependencies, but not package dependencies
// (as the latter will be covering code we're not reporting on).
api.Dependencies
.Where(dep => dep.Value == ProjectVersionValue)
.Select(dep => new XElement("FilterEntry", new XElement("ModuleMask", dep.Key)))
));
var attributeFilters = new XElement("AttributeFilters",
new XElement("AttributeFilterEntry",
"System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute"),
new XElement("AttributeFilterEntry", "System.Diagnostics.DebuggerNonUserCodeAttribute")
);
var output = new XElement("Output", $"../../../coverage/{Path.GetFileName(directory)}.dvcr");
var doc = new XElement("CoverageParams",
filters,
attributeFilters,
output
);
using (var stream = File.Create(Path.Combine(directory, "coverage.xml")))
{
doc.Save(stream);
}
}
static void GenerateDocumentationStub(string apiRoot, ApiMetadata api)
{
string file = Path.Combine(apiRoot, "docs", "index.md");
if (!File.Exists(file) && api.ProductName != null && api.ProductUrl != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(file));
File.WriteAllText(file,
@"{{title}}
{{description}}
{{version}}
{{installation}}
{{auth}}
# Getting started
{{client-classes}}
{{client-construction}}
");
Console.WriteLine($"Generated documentation stub for {api.Id}");
}
}
private static void GenerateCoverageFile(ApiMetadata api, string directory)
{
var targetExecutable = new XElement("TargetExecutable", "/Program Files/dotnet/dotnet.exe");
var targetArguments = new XElement("TargetArguments",
$"test --no-build -c Release");
var filters = new XElement("Filters", new XElement("IncludeFilters",
new XElement("FilterEntry", new XElement("ModuleMask", api.Id)),
// Allow tests to contribute coverage to project dependencies, but not package dependencies
// (as the latter will be covering code we're not reporting on).
api.Dependencies
.Where(dep => dep.Value == ProjectVersionValue)
.Select(dep => new XElement("FilterEntry", new XElement("ModuleMask", dep.Key)))
));
var attributeFilters = new XElement("AttributeFilters",
new XElement("AttributeFilterEntry",
"System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute"),
new XElement("AttributeFilterEntry", "System.Diagnostics.DebuggerNonUserCodeAttribute")
);
var output = new XElement("Output", $"../../../coverage/{Path.GetFileName(directory)}.dvcr");
var workingDir = new XElement("TargetWorkingDir", ".");
var doc = new XElement("CoverageParams",
targetExecutable,
targetArguments,
filters,
attributeFilters,
workingDir,
output
);
using (var stream = File.Create(Path.Combine(directory, "coverage.xml")))
{
doc.Save(stream);
}
}
static void GenerateDocumentationStub(string apiRoot, ApiMetadata api)
{
string file = Path.Combine(apiRoot, "docs", "index.md");
if (!File.Exists(file) && api.ProductName != null && api.ProductUrl != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(file));
File.WriteAllText(file,
@"{{title}}
{{description}}
{{installation}}
{{auth}}
# Getting started
TODO: Add a link to the client classes here, and introductory text.
# Sample code
TODO: Add snippet references here.
");
Console.WriteLine($"Generated documentation stub for {api.Id}");
}
}
static void GenerateSolutionFiles(string apiRoot, ApiMetadata api)
{
var projectDirectories = Directory.GetDirectories(apiRoot)
.Where(pd => Path.GetFileName(pd).StartsWith(api.Id))
.ToList();
HashSet <string> projects = new HashSet <string>();
// We want to include all the project files, and all the project references
// from those project files, being aware that the solution file is already one directory
// higher than the project file...
foreach (var dir in projectDirectories)
{
string projectName = Path.GetFileName(dir);
string projectFile = Path.Combine(dir, $"{projectName}.csproj");
if (File.Exists(projectFile))
{
projects.Add($"{projectName}/{projectName}.csproj");
XDocument doc = XDocument.Load(projectFile);
var projectReferences = doc.Descendants("ProjectReference")
.Select(x => x.Attribute("Include").Value.Replace("\\", "/"))
.Select(x => x.StartsWith("../") ? x.Substring(3) : x);
foreach (var reference in projectReferences)
{
projects.Add(reference);
}
}
}
var solutionFileName = $"{api.Id}.sln";
string fullFile = Path.Combine(apiRoot, solutionFileName);
string beforeHash = GetFileHash(fullFile);
if (!File.Exists(fullFile))
{
Processes.RunDotnet(apiRoot, "new", "sln", "-n", api.Id);
}
else
{
// Optimization: don't run "dotnet sln add" if we can find project entries for all the relevant project
// references already. This is crude, but speeds up the overall process significantly.
var projectLines = File.ReadAllLines(fullFile).Where(line => line.StartsWith("Project(")).ToList();
if (projects.Select(p => $"\"{p.Replace("/", "\\")}\"")
.All(expectedProject => projectLines.Any(pl => pl.Contains(expectedProject))))
{
return;
}
}
// It's much faster to run a single process than to run it once per project.
Processes.RunDotnet(apiRoot, new[] { "sln", solutionFileName, "add" }.Concat(projects).ToArray());
string afterHash = GetFileHash(fullFile);
if (beforeHash != afterHash)
{
Console.WriteLine($"{(beforeHash == null ? "Created" : "Modified")} solution file for {api.Id}");
}
}
private static void GenerateTestProject(ApiMetadata api, string directory, HashSet <string> apiNames, bool isForAnalyzers = false)
{
// Don't generate a project file if we've got a placeholder directory
if (Directory.GetFiles(directory, "*.cs", SearchOption.AllDirectories).Length == 0)
{
return;
}
var dependencies = new SortedList <string, string>(CommonTestDependencies);
if (isForAnalyzers)
{
dependencies.Remove("Google.Cloud.ClientTesting");
dependencies.Add("Google.Cloud.AnalyzersTesting", ProjectVersionValue);
}
dependencies.Add(api.Id, "project");
// Deliberately not using Add, so that a project can override the defaults.
foreach (var dependency in api.TestDependencies)
{
dependencies[dependency.Key] = dependency.Value;
}
var testTargetFrameworks = GetTestTargetFrameworks(api, isForAnalyzers);
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFrameworks", testTargetFrameworks),
new XElement("TargetFrameworks", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), AnyDesktopFramework.Replace(testTargetFrameworks, "")),
new XElement("LangVersion", "latest"),
new XElement("IsPackable", false),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("TreatWarningsAsErrors", true),
// 1701, 1702 and 1705 are disabled by default.
// xUnit2004 prevents Assert.Equal(true, value) etc, preferring Assert.True and Assert.False, but
// Assert.Equal is clearer (IMO) for comparing values rather than conditions.
// xUnit2013 prevents simple checks for the number of items in a collection
// AD0001 is the error for an analyzer throwing an exception. We can remove this when
// the fix for https://github.com/xunit/xunit/issues/1409 is in NuGet.
new XElement("NoWarn", "1701;1702;1705;xUnit2004;xUnit2013;AD0001")
);
string project = Path.GetFileName(directory);
var dependenciesElement = CreateDependenciesElement(project, dependencies, api.IsReleaseVersion, testProject: true, apiNames: apiNames);
// Include dotCover CLI tool for dotnet
dependenciesElement.Add(new XElement("DotNetCliToolReference",
new XAttribute("Include", "JetBrains.dotCover.CommandLineTools"),
new XAttribute("Version", "2018.2.0")));
// Allow test projects to use dynamic...
dependenciesElement.Add(new XElement("Reference",
new XAttribute("Condition", "'$(TargetFramework)' == 'net452'"),
new XAttribute("Include", "Microsoft.CSharp")));
// Test service... it keeps on getting added by Visual Studio, so let's just include it everywhere.
dependenciesElement.Add(new XElement("Service", new XAttribute("Include", "{82a7f48d-3b50-4b1e-b82e-3ada8210c358}")));
WriteProjectFile(api, directory, propertyGroup, dependenciesElement);
}
private static string GetPreferredCoverageFramework(ApiMetadata api)
{
var targetFrameworks = GetTestTargetFrameworks(api);
if (targetFrameworks.Contains(TargetFrameworkClassic))
{
return(TargetFrameworkClassic);
}
// Otherwise, return the first one found.
return(targetFrameworks.Split(';').FirstOrDefault());
}
private static void WriteProjectFile(
ApiMetadata api, string directory, XElement propertyGroup, XElement dependenciesItemGroup, XElement packingElement)
{
var file = Path.Combine(directory, $"{Path.GetFileName(directory)}.csproj");
string beforeHash = GetFileHash(file);
XElement doc;
// If the file already exists, load it and replace the elements (leaving any further PropertyGroup and ItemGroup elements).
// Make sure there's an appropriate import for stripping desktop builds on non-Windows platforms.
if (File.Exists(file))
{
doc = XElement.Load(file);
doc.Elements("Import").Where(x => (string)x.Attribute("Project") == @"..\..\StripDesktopOnNonWindows.xml").Remove();
doc.Elements("PropertyGroup").First().ReplaceWith(propertyGroup);
doc.Elements("ItemGroup").First().ReplaceWith(dependenciesItemGroup);
doc.Elements("ItemGroup").Where(x => (string)x.Attribute("Label") == DotnetPackInstructionsLabel).Remove();
doc.Elements("ItemGroup").First().AddAfterSelf(packingElement);
if (!doc.Elements("Import").Any(x => (string)x.Attribute("Project") == StripDesktopOnNonWindows))
{
doc.Add(new XElement("Import", new XAttribute("Project", StripDesktopOnNonWindows)));
}
}
// Otherwise, create a new one
else
{
doc = new XElement("Project",
new XAttribute("Sdk", "Microsoft.NET.Sdk"),
propertyGroup,
dependenciesItemGroup,
packingElement,
new XElement("Import", new XAttribute("Project", StripDesktopOnNonWindows))
);
}
// Don't use File.CreateText as that omits the byte order mark.
// While byte order marks are nasty, Visual Studio will add it back any time a project file is
// manually edited, so it's best if we follow suit.
using (var stream = File.Create(Path.Combine(directory, $"{Path.GetFileName(directory)}.csproj")))
{
doc.Save(stream);
}
string afterHash = GetFileHash(file);
if (beforeHash != afterHash)
{
Console.WriteLine($"{(beforeHash == null ? "Created" : "Modified")} project file {Path.GetFileName(file)}");
}
}
private static void GenerateSynthConfiguration(string apiRoot, ApiMetadata api)
{
if (api.Generator == GeneratorType.None)
{
return;
}
var synthFile = Path.Combine(apiRoot, "synth.py");
// Currently all APIs use the exact same synth file, so we can just replace it every time.
// We may need something more sophisticated in the future.
string content =
@"# GENERATED BY Google.Cloud.Tools.ProjectGenerator - DO NOT EDIT!
import json
import sys
from synthtool import shell
import synthtool.metadata
from pathlib import Path
AUTOSYNTH_MULTIPLE_COMMITS = True
# Parent of the script is the API-specific directory
# Parent of the API-specific directory is the apis directory
# Parent of the apis directory is the repo root
root = Path(__file__).parent.parent.parent
package = Path(__file__).parent.name
bash = '/bin/bash'
if sys.platform == 'win32':
bash = 'C:\\Program Files\\Git\\bin\\bash.exe'
shell.run(
(bash, 'generateapis.sh', '--check_compatibility', package),
cwd = root,
hide_output = False)
# Load the synth.metadata that generateapis.sh has written, and
# re-add all the sources for the in-memory version that synthtool
# is about to write out. This is a pretty ugly hack, but it works for now.
# (We assume every source is a git source.)
with open('synth.metadata') as generated_metadata_file:
generated_metadata = json.load(generated_metadata_file)
for source in generated_metadata['sources']:
synthtool.metadata.get().sources.add(git=source['git'])
";
File.WriteAllText(synthFile, content);
}
static void GenerateSolutionFiles(string apiRoot, ApiMetadata api)
{
var projectDirectories = Directory.GetDirectories(apiRoot)
.Where(pd => Path.GetFileName(pd).StartsWith(api.Id))
.ToList();
HashSet <string> projects = new HashSet <string>();
// We want to include all the project files, and all the project references
// from those project files, being aware that the solution file is already one directory
// higher than the project file...
foreach (var dir in projectDirectories)
{
string projectName = Path.GetFileName(dir);
string projectFile = Path.Combine(dir, $"{projectName}.csproj");
if (File.Exists(projectFile))
{
projects.Add($"{projectName}/{projectName}.csproj");
XDocument doc = XDocument.Load(projectFile);
var projectReferences = doc.Descendants("ProjectReference")
.Select(x => x.Attribute("Include").Value.Replace("\\", "/"))
.Select(x => x.StartsWith("../") ? x.Substring(3) : x);
foreach (var reference in projectReferences)
{
projects.Add(reference);
}
}
}
var solutionFileName = $"{api.Id}.sln";
string fullFile = Path.Combine(apiRoot, solutionFileName);
string beforeHash = GetFileHash(fullFile);
if (!File.Exists(fullFile))
{
Processes.RunDotnet(apiRoot, "new", "sln", "-n", api.Id);
}
// It's much faster to run a single process than to run it once per project.
Processes.RunDotnet(apiRoot, new[] { "sln", solutionFileName, "add" }.Concat(projects).ToArray());
string afterHash = GetFileHash(fullFile);
if (beforeHash != afterHash)
{
Console.WriteLine($"{(beforeHash == null ? "Created" : "Modified")} solution file for {api.Id}");
}
}
private static void GenerateSmokeTestProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
// Don't generate a project file if we've got a placeholder directory
if (Directory.GetFiles(directory, "*.cs", SearchOption.AllDirectories).Length == 0)
{
return;
}
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFramework", "netcoreapp2.1"),
new XElement("OutputType", "Exe"),
new XElement("IsPackable", false));
var dependenciesElement =
new XElement("ItemGroup",
CreateDependencyElement(Path.GetFileName(directory), api.Id, ProjectVersionValue, stableRelease: false, testProject: true, apiNames));
WriteProjectFile(api, directory, propertyGroup, dependenciesElement);
}
private static void GenerateTestProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
var dependencies = new SortedList <string, string>(CommonTestDependencies);
dependencies.Add(api.Id, "project");
// Deliberately not using Add, so that a project can override the defaults.
foreach (var dependency in api.TestDependencies)
{
dependencies[dependency.Key] = dependency.Value;
}
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFrameworks", GetTestTargetFrameworks(api)),
new XElement("TargetFrameworks", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), AnyDesktopFramework.Replace(GetTestTargetFrameworks(api), "")),
new XElement("LangVersion", "latest"),
new XElement("Features", "IOperation"),
new XElement("IsPackable", false),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("PublicSign", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), true),
new XElement("TreatWarningsAsErrors", true),
// 1701, 1702 and 1705 are disabled by default.
// xUnit2004 prevents Assert.Equal(true, value) etc, preferring Assert.True and Assert.False, but
// Assert.Equal is clearer (IMO) for comparing values rather than conditions.
// xUnit2013 prevents simple checks for the number of items in a collection
// AD0001 is the error for an analyzer throwing an exception. We can remove this when
// the fix for https://github.com/xunit/xunit/issues/1409 is in NuGet.
new XElement("NoWarn", "1701;1702;1705;xUnit2004;xUnit2013;AD0001")
);
string project = Path.GetFileName(directory);
var dependenciesElement = CreateDependenciesElement(project, dependencies, api.IsReleaseVersion, testProject: true, apiNames: apiNames);
// Allow test projects to use dynamic...
dependenciesElement.Add(new XElement("Reference",
new XAttribute("Condition", "'$(TargetFramework)' == 'net452'"),
new XAttribute("Include", "Microsoft.CSharp")));
// Test service... it keeps on getting added by Visual Studio, so let's just include it everywhere.
dependenciesElement.Add(new XElement("Service", new XAttribute("Include", "{82a7f48d-3b50-4b1e-b82e-3ada8210c358}")));
WriteProjectFile(api, directory, propertyGroup, dependenciesElement, null);
}
static int Main()
{
try
{
var root = DirectoryLayout.DetermineRootDirectory();
foreach (var api in ApiMetadata.LoadApis())
{
GenerateProjects(Path.Combine(root, "apis", api.Id), api);
}
foreach (var api in ApiMetadata.LoadApis())
{
GenerateSolutionFiles(Path.Combine(root, "apis", api.Id), api);
}
return(0);
}
catch (Exception e)
{
Console.WriteLine($"Failed: {e}");
return(1);
}
}
private static void GenerateSampleProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
// Don't generate a project file if we've got a placeholder directory
if (Directory.GetFiles(directory, "*.cs", SearchOption.AllDirectories).Length == 0)
{
return;
}
var dependencies = new SortedList <string, string>(CommonSampleDependencies);
dependencies.Add(api.Id, "project");
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFramework", "netcoreapp2.1"),
new XElement("OutputType", "Exe"),
new XElement("LangVersion", "latest"),
new XElement("IsPackable", false));
string project = Path.GetFileName(directory);
var dependenciesElement = CreateDependenciesElement(project, dependencies, api.IsReleaseVersion, testProject: true, apiNames: apiNames);
WriteProjectFile(api, directory, propertyGroup, dependenciesElement);
}
private static void GenerateTestProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
var dependencies = new SortedList <string, string>(CommonTestDependencies);
dependencies.Add(api.Id, "project");
// Deliberately not using Add, so that a project can override the defaults.
foreach (var dependency in api.TestDependencies)
{
dependencies[dependency.Key] = dependency.Value;
}
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFrameworks", GetTestTargetFrameworks(api)),
new XElement("LangVersion", "latest"),
new XElement("Features", "IOperation"),
new XElement("IsPackable", false),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("PublicSign", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), true),
new XElement("TreatWarningsAsErrors", true),
// 1701, 1702 and 1705 are disabled by default.
// 4014 is required as snippets for streaming samples call Task.Run and don't await the result.
// See https://github.com/googleapis/toolkit/issues/1271 - when that's fixed, we can remove this.
new XElement("NoWarn", "1701;1702;1705;4014")
);
string project = Path.GetFileName(directory);
var dependenciesElement = CreateDependenciesElement(project, dependencies, api.IsReleaseVersion, testProject: true, apiNames: apiNames);
// Allow test projects to use dynamic...
dependenciesElement.Add(new XElement("Reference",
new XAttribute("Condition", "'$(TargetFramework)' == 'net452'"),
new XAttribute("Include", "Microsoft.CSharp")));
// Test service... it keeps on getting added by Visual Studio, so let's just include it everywhere.
dependenciesElement.Add(new XElement("Service", new XAttribute("Include", "{82a7f48d-3b50-4b1e-b82e-3ada8210c358}")));
WriteProjectFile(api, directory, propertyGroup, dependenciesElement, null);
}
/// <summary>
/// Generates a metadata file (currently .repo-metadata.json; may change name later) with
/// all the information that language-agnostic tools require.
/// </summary>
private static void GenerateMetadataFile(string apiRoot, ApiMetadata api)
{
var version = api.StructuredVersion;
// Version "1.0.0-beta00" hasn't been released at all, so we don't have a package to talk about.
// TODO: Check that this is actually appropriate.
if ((version.Prerelease ?? "").EndsWith("00") && version.Major == 1 && version.Minor == 0)
{
return;
}
string releaseLevel =
// If it's not a prerelease now, or it's ever got to 1.0, it's generally "ga"
version.Major > 1 || version.Minor > 0 || version.Prerelease == null ? "ga"
: version.Prerelease.StartsWith("beta") ? "beta" : "alpha";
var metadata = new
{
distribution_name = api.Id,
release_level = releaseLevel
};
string json = JsonConvert.SerializeObject(metadata, Formatting.Indented);
File.WriteAllText(Path.Combine(apiRoot, ".repo-metadata.json"), json);
}
private static void WriteProjectFile(
ApiMetadata api, string directory, XElement propertyGroup, XElement dependenciesItemGroup)
{
var file = Path.Combine(directory, $"{Path.GetFileName(directory)}.csproj");
string beforeHash = GetFileHash(file);
XElement doc;
// If the file already exists, load it and replace the elements (leaving any further PropertyGroup and ItemGroup elements).
if (File.Exists(file))
{
doc = XElement.Load(file);
doc.Elements("PropertyGroup").First().ReplaceWith(propertyGroup);
doc.Elements("ItemGroup").First().ReplaceWith(dependenciesItemGroup);
}
// Otherwise, create a new one
else
{
doc = new XElement("Project",
new XAttribute("Sdk", "Microsoft.NET.Sdk"),
propertyGroup,
dependenciesItemGroup);
}
// Don't use File.CreateText as that omits the byte order mark.
// While byte order marks are nasty, Visual Studio will add it back any time a project file is
// manually edited, so it's best if we follow suit.
using (var stream = File.Create(Path.Combine(directory, $"{Path.GetFileName(directory)}.csproj")))
{
doc.Save(stream);
}
string afterHash = GetFileHash(file);
if (beforeHash != afterHash)
{
Console.WriteLine($"{(beforeHash == null ? "Created" : "Modified")} project file {Path.GetFileName(file)}");
}
}
static int Main()
{
try
{
ValidateCommonHiddenProductionDependencies();
var root = DirectoryLayout.DetermineRootDirectory();
var apis = ApiMetadata.LoadApis();
Console.WriteLine($"API catalog contains {apis.Count} entries");
// Now we know we can parse the API catalog, let's reformat it.
ReformatApiCatalog();
RewriteReadme(apis);
RewriteDocsRootIndex(apis);
HashSet <string> apiNames = new HashSet <string>(apis.Select(api => api.Id));
foreach (var api in apis)
{
var path = Path.Combine(root, "apis", api.Id);
GenerateProjects(path, api, apiNames);
GenerateSolutionFiles(path, api);
GenerateDocumentationStub(path, api);
GenerateSynthConfiguration(path, api);
GenerateMetadataFile(path, api);
}
return(0);
}
catch (UserErrorException e)
{
Console.WriteLine($"Configuration error: {e.Message}");
return(1);
}
catch (Exception e)
{
Console.WriteLine($"Failed: {e}");
return(1);
}
}
private static string GetTestTargetFrameworks(ApiMetadata api, bool isForAnalyzers) =>
isForAnalyzers
? AnalyzersTestTargetFramework
: api.TestTargetFrameworks ?? api.TargetFrameworks ?? DefaultTestTargetFrameworks;
private static void GenerateMainProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
if (api.Version == null)
{
throw new UserErrorException($"No version specified for {api.Id}");
}
string targetFrameworks = api.TargetFrameworks;
SortedList <string, string> dependencies;
if (api.Type == ApiType.Analyzers)
{
if (targetFrameworks != AnalyzersTargetFramework)
{
throw new UserErrorException($"Analyzers are expected to use {AnalyzersTargetFramework}");
}
dependencies = new SortedList <string, string>(CommonAnalyzerDependencies);
// Note: If support is added here for using additional dependencies, we need to resolve
// the packaging issues and make sure the onus won't be on the user to add the
// dependency references.
}
else
{
dependencies = new SortedList <string, string>(CommonHiddenProductionDependencies);
switch (api.Type)
{
case ApiType.Rest:
dependencies.Add("Google.Api.Gax.Rest", DefaultVersionValue);
targetFrameworks = targetFrameworks ?? DefaultRestTargetFrameworks;
break;
case ApiType.Grpc:
dependencies.Add("Google.Api.Gax.Grpc", DefaultVersionValue);
dependencies.Add("Grpc.Core", DefaultVersionValue);
targetFrameworks = targetFrameworks ?? DefaultGrpcTargetFrameworks;
break;
}
// Deliberately not using Add, so that a project can override the defaults.
// In particular, stable releases *must* override versions of GRPC and GAX.
foreach (var dependency in api.Dependencies)
{
dependencies[dependency.Key] = dependency.Value;
}
}
var propertyGroup = new XElement("PropertyGroup",
// Build-related properties
new XElement("Version", api.Version), // TODO: Version, or VersionPrefix/VersionSuffix?
new XElement("TargetFrameworks", targetFrameworks),
new XElement("TargetFrameworks", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), AnyDesktopFramework.Replace(targetFrameworks, "")),
new XElement("LangVersion", "latest"),
new XElement("GenerateDocumentationFile", api.Type != ApiType.Analyzers),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("Deterministic", true),
new XElement("TreatWarningsAsErrors", true),
// Package-related properties
new XElement("Description", api.Description),
new XElement("PackageTags", string.Join(";", api.Tags.Concat(new[] { "Google", "Cloud" }))),
new XElement("Copyright", $"Copyright {DateTime.UtcNow.Year} Google LLC"),
new XElement("Authors", "Google Inc."),
new XElement("PackageIconUrl", "https://cloud.google.com/images/gcp-icon-64x64.png"),
new XElement("PackageLicenseFile", "LICENSE"),
new XElement("PackageProjectUrl", "https://github.com/googleapis/google-cloud-dotnet"),
new XElement("RepositoryType", "git"),
new XElement("RepositoryUrl", "https://github.com/googleapis/google-cloud-dotnet")
);
if (dependencies.ContainsKey(GrpcPackage))
{
propertyGroup.Add(new XElement("CodeAnalysisRuleSet", "..\\..\\..\\grpc.ruleset"));
}
var dependenciesElement = CreateDependenciesElement(api.Id, dependencies, api.IsReleaseVersion, testProject: false, apiNames: apiNames);
// Pack the license file; this element isn't a dependency, but it still belongs in an ItemGroup...
dependenciesElement.Add(new XElement("None",
new XAttribute("Include", "../../../LICENSE"),
new XAttribute("Pack", true),
new XAttribute("PackagePath", ""))); // Note: not $(PackageLicenseFile) as suggested in docs, due to us using a file with no extension
if (api.Type == ApiType.Analyzers)
{
propertyGroup.Add(new XElement("IncludeBuildOutput", false));
void AddPackFile(string includePath, string packagePath)
{
dependenciesElement.Add(
new XElement("None",
new XAttribute("Include", includePath),
new XAttribute("Pack", "true"),
new XAttribute("PackagePath", packagePath),
new XAttribute("Visible", "false")));
}
AddPackFile(
$"$(OutputPath)\\{AnalyzersTargetFramework}\\$(AssemblyName).dll",
"analyzers/dotnet/cs");
// Add install scripts as per
// https://docs.microsoft.com/en-us/nuget/reference/analyzers-conventions#install-and-uninstall-scripts
// Name each file rather than using a wildcard so 'dotnet pack' will error out if the files are missing
// for some reason.
AddPackFile(
@"..\..\..\analyzerScripts\install.ps1",
"tools");
AddPackFile(
@"..\..\..\analyzerScripts\uninstall.ps1",
"tools");
}
WriteProjectFile(api, directory, propertyGroup, dependenciesElement);
}
private static void GenerateMainProject(ApiMetadata api, string directory)
{
if (api.Version == null)
{
throw new Exception($"No version specified for {api.Id}");
}
string targetFrameworks = api.TargetFrameworks;
var dependencies = new SortedList <string, string>(api.Dependencies)
{
{ ConfigureAwaitAnalyzer, "1.0.0-beta4" },
{ SourceLinkPackage, "2.1.2" }
};
// If Grpc.Core is ever specified explicitly (e.g. for "other" projects),
// but without a version number, fill it in.
if (dependencies.ContainsKey("Grpc.Core") && dependencies["Grpc.Core"] == "")
{
dependencies["Grpc.Core"] = GrpcVersion;
}
switch (api.Type)
{
case "rest":
dependencies.Add("Google.Api.Gax.Rest", api.IsReleaseVersion ? StableGaxVersion : PrereleaseGaxVersion);
targetFrameworks = targetFrameworks ?? "netstandard1.3;net45";
break;
case "grpc":
dependencies.Add("Google.Api.Gax.Grpc", api.IsReleaseVersion ? StableGaxVersion : PrereleaseGaxVersion);
dependencies.Add("Grpc.Core", GrpcVersion);
targetFrameworks = targetFrameworks ?? "netstandard1.5;net45";
break;
}
var propertyGroup = new XElement("PropertyGroup",
// Build-related properties
new XElement("Version", api.Version), // TODO: Version, or VersionPrefix/VersionSuffix?
new XElement("TargetFrameworks", targetFrameworks),
new XElement("Features", "IOperation"),
new XElement("GenerateDocumentationFile", true),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("Deterministic", true),
new XElement("PublicSign", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), true),
new XElement("TreatWarningsAsErrors", true),
// Package-related properties
new XElement("Description", api.Description),
new XElement("PackageTags", string.Join(";", api.Tags.Concat(new[] { "Google", "Cloud" }))),
new XElement("Copyright", "Copyright 2017 Google Inc."),
new XElement("Authors", "Google Inc."),
new XElement("PackageIconUrl", "https://cloud.google.com/images/gcp-icon-64x64.png"),
new XElement("PackageLicenseUrl", "http://www.apache.org/licenses/LICENSE-2.0"),
new XElement("PackageProjectUrl", "https://github.com/GoogleCloudPlatform/google-cloud-dotnet"),
new XElement("RepositoryType", "git"),
new XElement("RepositoryUrl", "https://github.com/GoogleCloudPlatform/google-cloud-dotnet")
);
var packingElement = new XElement("ItemGroup",
new XAttribute("Label", DotnetPackInstructionsLabel),
targetFrameworks.Split(';').Select(tfm => new XElement("Content",
new XAttribute("Include", $@"$(OutputPath){tfm}\$(PackageId).pdb"),
new XElement("Pack", true),
new XElement("PackagePath", $"lib/{tfm}")
))
);
WriteProjectFile(api, directory, propertyGroup, CreateDependenciesElement(dependencies, api.IsReleaseVersion), packingElement);
}
private static void GenerateMainProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
if (api.Version == null)
{
throw new UserErrorException($"No version specified for {api.Id}");
}
string targetFrameworks = api.TargetFrameworks;
SortedList <string, string> dependencies;
if (api.Type == ApiType.Analyzers)
{
if (targetFrameworks != AnalyzersTargetFramework)
{
throw new UserErrorException($"Analyzers are expected to use {AnalyzersTargetFramework}");
}
dependencies = new SortedList <string, string>(CommonAnalyzerDependencies, StringComparer.Ordinal);
// Note: If support is added here for using additional dependencies, we need to resolve
// the packaging issues and make sure the onus won't be on the user to add the
// dependency references.
}
else
{
dependencies = new SortedList <string, string>(CommonHiddenProductionDependencies, StringComparer.Ordinal);
switch (api.Type)
{
case ApiType.Rest:
dependencies.Add("Google.Api.Gax.Rest", DefaultVersionValue);
targetFrameworks = targetFrameworks ?? DefaultRestTargetFrameworks;
break;
case ApiType.Grpc:
dependencies.Add("Google.Api.Gax.Grpc.GrpcCore", DefaultVersionValue);
dependencies.Add("Grpc.Core", DefaultVersionValue);
targetFrameworks = targetFrameworks ?? DefaultGrpcTargetFrameworks;
break;
}
// Deliberately not using Add, so that a project can override the defaults.
// In particular, stable releases *must* override versions of GRPC and GAX.
foreach (var dependency in api.Dependencies)
{
dependencies[dependency.Key] = dependency.Value;
}
}
var propertyGroup = new XElement("PropertyGroup",
// Build-related properties
new XElement("Version", api.Version), // TODO: Version, or VersionPrefix/VersionSuffix?
new XElement("TargetFrameworks", targetFrameworks),
new XElement("GenerateDocumentationFile", api.Type != ApiType.Analyzers),
// Package-related properties
new XElement("Description", api.Description),
new XElement("PackageTags", string.Join(";", api.Tags.Concat(new[] { "Google", "Cloud" }))),
new XElement("Copyright", $"Copyright {DateTime.UtcNow.Year} Google LLC")
);
if (dependencies.ContainsKey(GrpcPackage))
{
propertyGroup.Add(new XElement("CodeAnalysisRuleSet", "..\\..\\..\\grpc.ruleset"));
}
var dependenciesElement = CreateDependenciesElement(api.Id, dependencies, api.IsReleaseVersion, testProject: false, apiNames: apiNames);
if (api.Type == ApiType.Analyzers)
{
propertyGroup.Add(new XElement("IncludeBuildOutput", false));
void AddPackFile(string includePath, string packagePath)
{
dependenciesElement.Add(
new XElement("None",
new XAttribute("Include", includePath),
new XAttribute("Pack", "true"),
new XAttribute("PackagePath", packagePath),
new XAttribute("Visible", "false")));
}
AddPackFile(
$"$(OutputPath)\\{AnalyzersTargetFramework}\\$(AssemblyName).dll",
"analyzers/dotnet/cs");
// Add install scripts as per
// https://docs.microsoft.com/en-us/nuget/reference/analyzers-conventions#install-and-uninstall-scripts
// Name each file rather than using a wildcard so 'dotnet pack' will error out if the files are missing
// for some reason.
AddPackFile(
@"..\..\..\analyzerScripts\install.ps1",
"tools");
AddPackFile(
@"..\..\..\analyzerScripts\uninstall.ps1",
"tools");
}
WriteProjectFile(api, directory, propertyGroup, dependenciesElement);
}
private static void GenerateMainProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
if (api.Version == null)
{
throw new UserErrorException($"No version specified for {api.Id}");
}
string targetFrameworks = api.TargetFrameworks;
var dependencies = new SortedList <string, string>(CommonHiddenProductionDependencies);
switch (api.Type)
{
case "rest":
dependencies.Add("Google.Api.Gax.Rest", DefaultVersionValue);
targetFrameworks = targetFrameworks ?? DefaultRestTargetFrameworks;
break;
case "grpc":
dependencies.Add("Google.Api.Gax.Grpc", DefaultVersionValue);
dependencies.Add("Grpc.Core", DefaultVersionValue);
targetFrameworks = targetFrameworks ?? DefaultGrpcTargetFrameworks;
break;
}
// Deliberately not using Add, so that a project can override the defaults.
// In particular, stable releases *must* override versions of GRPC and GAX.
foreach (var dependency in api.Dependencies)
{
dependencies[dependency.Key] = dependency.Value;
}
var propertyGroup = new XElement("PropertyGroup",
// Build-related properties
new XElement("Version", api.Version), // TODO: Version, or VersionPrefix/VersionSuffix?
new XElement("TargetFrameworks", targetFrameworks),
new XElement("TargetFrameworks", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), AnyDesktopFramework.Replace(targetFrameworks, "")),
new XElement("LangVersion", "latest"),
new XElement("Features", "IOperation"),
new XElement("GenerateDocumentationFile", true),
new XElement("AssemblyOriginatorKeyFile", "../../GoogleApis.snk"),
new XElement("SignAssembly", true),
new XElement("Deterministic", true),
new XElement("PublicSign", new XAttribute("Condition", " '$(OS)' != 'Windows_NT' "), true),
new XElement("TreatWarningsAsErrors", true),
// Package-related properties
new XElement("Description", api.Description),
new XElement("PackageTags", string.Join(";", api.Tags.Concat(new[] { "Google", "Cloud" }))),
new XElement("Copyright", "Copyright 2017 Google Inc."),
new XElement("Authors", "Google Inc."),
new XElement("PackageIconUrl", "https://cloud.google.com/images/gcp-icon-64x64.png"),
new XElement("PackageLicenseUrl", "http://www.apache.org/licenses/LICENSE-2.0"),
new XElement("PackageProjectUrl", "https://github.com/GoogleCloudPlatform/google-cloud-dotnet"),
new XElement("RepositoryType", "git"),
new XElement("RepositoryUrl", "https://github.com/GoogleCloudPlatform/google-cloud-dotnet")
);
if (dependencies.ContainsKey(GrpcPackage))
{
propertyGroup.Add(new XElement("CodeAnalysisRuleSet", "..\\..\\..\\grpc.ruleset"));
}
var packingElement = new XElement("ItemGroup",
new XAttribute("Label", DotnetPackInstructionsLabel),
targetFrameworks.Split(';').Select(tfm => new XElement("Content",
new XAttribute("Include", $@"$(OutputPath){tfm}\$(PackageId).pdb"),
new XElement("Pack", true),
new XElement("PackagePath", $"lib/{tfm}")
))
);
var dependenciesElement = CreateDependenciesElement(api.Id, dependencies, api.IsReleaseVersion, testProject: false, apiNames: apiNames);
WriteProjectFile(api, directory, propertyGroup, dependenciesElement, packingElement);
}
private static string GetTestTargetFrameworks(ApiMetadata api) => api.TestTargetFrameworks ?? api.TargetFrameworks ?? DefaultTestTargetFrameworks;
