private static async Task <int> MainAsync(string[] args)
{
var client = new GitHubClient(new ProductHeaderValue(ApplicationName))
{
Credentials = new Octokit.Credentials(args[0])
};
var commit = await FetchRemoteCommitAsync(client);
ValidateLocalRepository(commit);
var apis = ApiMetadata.LoadApis();
var newReleases = await ComputeNewReleasesAsync(client, apis);
ValidateChanges(newReleases);
if (!ConfirmReleases(apis, newReleases))
{
return(0);
}
await MakeReleasesAsync(client, newReleases, commit);
Console.WriteLine();
Console.WriteLine($"Release tags created. Please wait for emails confirming the automated build and push process.");
Console.WriteLine($"For a manual release, run ./buildrelease.sh {commit}");
return(0);
}
/// <summary>
/// Get meta data for the last Pelion Device Management API call
/// </summary>
/// <returns><see cref="ApiMetadata"/></returns>
public static ApiMetadata GetLastApiMetadata()
{
var lastMds = mds.Client.Configuration.Default.ApiClient.LastApiResponse.LastOrDefault()?.Headers?.Where(m => m.Name == "Date")?.Select(d => DateTime.Parse(d.Value.ToString()))?.FirstOrDefault();
var lastStats = statistics.Client.Configuration.Default.ApiClient.LastApiResponse.LastOrDefault()?.Headers?.Where(m => m.Name == "Date")?.Select(d => DateTime.Parse(d.Value.ToString()))?.FirstOrDefault();
return(Nullable.Compare(lastMds, lastStats) > 0 ? ApiMetadata.Map(mds.Client.Configuration.Default.ApiClient.LastApiResponse.LastOrDefault()) : ApiMetadata.Map(statistics.Client.Configuration.Default.ApiClient.LastApiResponse.LastOrDefault()));
}
/// <summary>
/// Checks the compatibility of the locally-built API against a version on NuGet.
/// This assumes the local package has already been built and is up-to-date.
/// </summary>
private static Level CheckCompatibility(ApiMetadata api, StructuredVersion version)
{
Console.WriteLine($"Differences from {version}");
// TODO: Remove this try/catch when *everything* has a previous minor version on netstandard2.0.
AssemblyDefinition oldMetadata;
try
{
oldMetadata = Assemblies.LoadPackageAsync(api.Id, version.ToString(), null, null).Result;
}
catch (Exception e)
{
Console.WriteLine($"Unable to load {api.Id} version {version} from NuGet. Some possible causes:");
Console.WriteLine("- Package was pre-netstandard2.0");
Console.WriteLine("- Package was never published");
Console.WriteLine("- nuget.org failure");
Console.WriteLine($"Exception message: {e.Message}");
Console.WriteLine($"Returning 'identical' as the change level; please check carefully before release.");
return(Level.Identical);
}
var sourceAssembly = Path.Combine(DirectoryLayout.ForApi(api.Id).SourceDirectory, api.Id, "bin", "Release", "netstandard2.0", $"{api.Id}.dll");
var newMetadata = Assemblies.LoadFile(sourceAssembly);
var diff = Assemblies.Compare(oldMetadata, newMetadata, null);
diff.PrintDifferences(Level.Major, FormatDetail.Brief);
diff.PrintDifferences(Level.Minor, FormatDetail.Brief);
Console.WriteLine($"Diff level: {diff.Level}");
Console.WriteLine();
return(diff.Level);
}
private static void MaybeShowLagging(List <Tag> allTags, ApiMetadata api)
{
var currentVersion = api.StructuredVersion;
// Skip anything that is naturally pre-release (in the API), or where the current release is GA already.
if (!api.CanHaveGaRelease || currentVersion.Prerelease is null)
{
return;
}
// Find all the existing prereleases for the expected "next GA" release.
var expectedGa = StructuredVersion.FromMajorMinorPatch(currentVersion.Major, currentVersion.Minor, currentVersion.Patch, prerelease: null);
string expectedGaPrefix = $"{api.Id}-{expectedGa}";
var matchingReleaseTags = allTags.Where(tag => tag.FriendlyName.StartsWith(expectedGaPrefix, StringComparison.Ordinal)).ToList();
// Skip if we haven't even released the current prerelease.
if (matchingReleaseTags.Count == 0)
{
return;
}
var latest = GitHelpers.GetDate(matchingReleaseTags.First());
var earliest = GitHelpers.GetDate(matchingReleaseTags.Last());
string dateRange = latest == earliest
? $"{latest:yyyy-MM-dd}"
: $"{earliest:yyyy-MM-dd} - {latest:yyyy-MM-dd}";
Console.WriteLine($"{api.Id,-50}{api.Version,-20}{dateRange}");
}
/// <summary>
/// Generates the docfx-devsite.json file used to generate just the metadata for DevSite.
/// </summary>
private static void CreateDevsiteDocfxJson(ApiCatalog catalog, ApiMetadata rootApi, string outputDirectory)
{
// Pick whichever framework is listed first. (This could cause problems if a dependency
// doesn't target the given framework, but that seems unlikely.)
// Default to netstandard2.0 if nothing is listed.
string targetFramework = rootApi.TargetFrameworks?.Split(';').First() ?? "netstandard2.0";
var json = new JObject
{
["metadata"] = new JArray {
new JObject
{
["src"] = new JObject
{
["files"] = new JArray {
$"{rootApi.Id}/{rootApi.Id}.csproj"
},
["cwd"] = $"../../../apis/{rootApi.Id}"
},
["dest"] = "obj/bareapi",
["filter"] = "filterConfig.yml",
["properties"] = new JObject {
["TargetFramework"] = targetFramework
}
},
}
};
File.WriteAllText(Path.Combine(outputDirectory, "docfx-devsite.json"), json.ToString());
}
public MetadataResolver()
{
var assembly = typeof(T).Assembly;
var asmName = assembly.GetName();
_metadata = new ApiMetadata(asmName.Name, asmName.Version.ToString());
}
private static async Task <int> MainAsync(string[] args)
{
var client = new GitHubClient(new ProductHeaderValue(ApplicationName))
{
Credentials = new Octokit.Credentials(args[0])
};
var commit = await FetchRemoteCommitAsync(client);
ValidateLocalRepository(commit);
var apis = ApiMetadata.LoadApis();
var newReleases = ComputeNewReleasesAsync(apis);
ValidateChanges(newReleases);
if (!ConfirmReleases(newReleases))
{
return(0);
}
await MakeReleasesAsync(client, newReleases, commit);
Console.WriteLine();
Console.WriteLine($"Release tags created. Please start the Kokoro release job with commit hash \"{commit.Sha}\" and wait for an email with the result.");
Console.WriteLine($"For a manual release, run ./buildrelease.sh {commit.Sha}");
return(0);
}
private static void ModifyForDevSite(ApiMetadata api, JObject obj)
{
// We won't build the metadata, so let's remove it.
obj.Remove("metadata");
var build = (JObject)obj["build"];
var globalMetadata = (JObject)build["globalMetadata"];
globalMetadata["_disableNavbar"] = true;
globalMetadata["_disable"] = true;
globalMetadata["_disableBreadcrumb"] = true;
globalMetadata["_enableSearch"] = false;
globalMetadata["_disableToc"] = true;
globalMetadata["_disableSideFilter"] = true;
globalMetadata["_disableAffix"] = true;
globalMetadata["_disableFooter"] = true;
// First pass at guessing the root path to use. We will want to infer from other things, but if
// we get it wrong for now, it won't matter as it's not public.
string productUrl = api.ProductUrl ?? "";
string productFamily = productUrl.StartsWith("https://cloud.google.com/")
? productUrl.Split('/')[3]
: "unknown";
globalMetadata["_rootPath"] = $"/dotnet/reference/{productFamily}";
build["template"][1] = "../../../third_party/docfx/templates/devsite";
build["dest"] = "devsite";
}
private static string CreateClientClassesDocumentation(ApiMetadata api)
{
if (api.Type != "grpc")
{
return("FIXME"); // No automatic templating for this API
}
var layout = DirectoryLayout.ForApi(api.Id);
var packageSource = Path.Combine(layout.SourceDirectory, api.Id);
var sourceFiles = Directory.GetFiles(packageSource, "*Client.cs");
// TODO: Find a more robust way of detecting the clients.
var clients = sourceFiles
.Where(file => File.ReadAllText(file).Contains(": ServiceSettingsBase")) // Check it contains a generated client
.Select(file => Path.GetFileName(file)) // Just the file name, not full path
.Select(file => file.Substring(0, file.Length - 3)) // Trim .cs
.OrderBy(client => client)
.Select(client => $"[{client}](obj/api/{api.Id}.{client}.yml)") // Markdown link to API doc
.ToList();
switch (clients.Count)
{
case 0: return("FIXME"); // No automatic templating for this API
case 1: return($"All operations are performed through {clients[0]}.");
default:
var list = string.Join("\r\n", clients.Select(client => $"- {client}"));
return($"All operations are performed through the following client classes:\r\n\r\n{list}");
}
}
private static int MainImpl(string[] args)
{
if (args.Length != 1)
{
throw new UserErrorException("Please specify the API name");
}
string api = args[0];
var layout = DirectoryLayout.ForApi(api);
var apiMetadata = ApiMetadata.LoadApis().FirstOrDefault(x => x.Id == api);
if (apiMetadata == null)
{
throw new UserErrorException($"Unable to load API metadata from apis.json for {api}");
}
string output = layout.DocsOutputDirectory;
if (Directory.Exists(output))
{
Directory.Delete(output, true);
}
Directory.CreateDirectory(output);
var apiDirectory = layout.SourceDirectory;
var projects = Project.LoadProjects(apiDirectory).ToList();
CreateDocfxJson(api, projects, output);
CopyAndGenerateArticles(apiMetadata, layout.DocsSourceDirectory, output);
CreateToc(api, output);
return(0);
}
static void GenerateDocumentationStub(string apiRoot, ApiMetadata api)
{
string file = Path.Combine(apiRoot, "docs", "index.md");
if (File.Exists(file))
{
return;
}
Directory.CreateDirectory(Path.GetDirectoryName(file));
string stub = api.ProductName != null && api.ProductUrl != null ?
@"{{title}}
{{description}}
{{version}}
{{installation}}
{{auth}}
# Getting started
{{client-classes}}
{{client-construction}}
"
: "{{non-product-stub}}";
File.WriteAllText(file, stub);
Console.WriteLine($"Generated documentation stub for {api.Id}");
}
/// <summary>
/// Generates a metadata file (currently .repo-metadata.json; may change name later) with
/// all the information that language-agnostic tools require.
/// </summary>
public static void GenerateMetadataFile(string apiRoot, ApiMetadata api)
{
string metadataPath = Path.Combine(apiRoot, ".repo-metadata.json");
var version = api.StructuredVersion;
string versionBasedReleaseLevel =
// Version "1.0.0-beta00" hasn't been released at all, so we don't have a package to talk about.
(version.Prerelease ?? "").EndsWith("00") && version.Major == 1 && version.Minor == 0 ? "none"
// If it's not a prerelease now, or it's ever got to 1.0, it's generally "ga"
: version.Major > 1 || version.Minor > 0 || version.Prerelease == null ? "ga"
: version.Prerelease.StartsWith("beta") ? "beta" : "alpha";
string releaseLevel = api.ReleaseLevelOverride ?? versionBasedReleaseLevel;
if (releaseLevel == "none")
{
// If we have temporarily set the version to (say) beta01 and then reset it to beta00,
// make sure we don't have an obsolete metadata file.
File.Delete(metadataPath);
return;
}
var metadata = new
{
distribution_name = api.Id,
release_level = releaseLevel,
client_documentation = ApiMetadata.IsCloudPackage(api.Id) ? $"https://cloud.google.com/dotnet/docs/reference/{api.Id}/latest" : $"https://googleapis.dev/dotnet/{api.Id}/latest",
library_type = api.EffectiveMetadataType
};
string json = JsonConvert.SerializeObject(metadata, Formatting.Indented);
File.WriteAllText(metadataPath, json);
}
/// <summary>
/// Updates the dependencies in an API for known packages, but only if the default
/// version is later than the current one, with the same major version number.
/// </summary>
public static void UpdateDependencies(ApiCatalog catalog, ApiMetadata api)
{
// Update any previously-defaulted versions to be explicit, if the new version is GA.
// (This only affects production dependencies, so is not performed in UpdateDependencyDictionary.)
// Implicit dependencies are always present in DefaultPackageVersions, so we don't need to worry about
// "internal" dependencies.
if (api.IsReleaseVersion && PackageTypeToImplicitDependencies.TryGetValue(api.Type, out var implicitDependencies))
{
foreach (var implicitDependency in implicitDependencies)
{
if (!api.Dependencies.ContainsKey(implicitDependency))
{
api.Dependencies[implicitDependency] = DefaultPackageVersions[implicitDependency];
}
}
}
UpdateDependencyDictionary(api.Dependencies, "dependencies");
UpdateDependencyDictionary(api.TestDependencies, "testDependencies");
void UpdateDependencyDictionary(SortedDictionary <string, string> dependencies, string jsonName)
{
if (dependencies.Count == 0)
{
return;
}
// We want to update any dependencies to "external" packages as listed in DefaultPackageVersions,
// but also "internal" packages such as Google.LongRunning.
Dictionary <string, string> allDefaultPackageVersions = DefaultPackageVersions
.Concat(catalog.Apis.Select(api => new KeyValuePair <string, string>(api.Id, api.Version)))
.ToDictionary(pair => pair.Key, pair => pair.Value);
foreach (var package in dependencies.Keys.ToList())
{
if (allDefaultPackageVersions.TryGetValue(package, out var defaultVersion))
{
var currentVersion = dependencies[package];
if (currentVersion == DefaultVersionValue ||
currentVersion == ProjectVersionValue ||
defaultVersion == currentVersion)
{
continue;
}
var structuredDefaultVersion = StructuredVersion.FromString(defaultVersion);
var structuredCurrentVersion = StructuredVersion.FromString(currentVersion);
if (structuredDefaultVersion.CompareTo(structuredCurrentVersion) > 0 &&
structuredDefaultVersion.Major == structuredCurrentVersion.Major)
{
dependencies[package] = defaultVersion;
}
}
}
if (api.Json is object)
{
api.Json[jsonName] = new JObject(dependencies.Select(pair => new JProperty(pair.Key, pair.Value)));
}
}
}
/// <summary>
/// Extremely crude templating, but just enough for now... it replaces the following tokens:
/// {{title}}: Markdown for the page title with the API ID
/// {{description}}: Markdown for the API description
/// {{installation}}: Markdown for the installation section
/// {{auth}}: Markdown for authentication instructions
/// </summary>
private static string TransformDocTemplate(ApiMetadata api, string text)
{
string title = $"# {api.Id}";
string description = $"`{api.Id}` is a.NET client library for the [{api.ProductName} API]({api.ProductUrl}).";
string installation =
$@"# Installation
Install the `{api.Id}` package from NuGet. Add it to
your project in the normal way (for example by right-clicking on the
project in Visual Studio and choosing ""Manage NuGet Packages..."").";
if (!api.IsReleaseVersion)
{
installation += @"
Please ensure you enable pre-release packages(for example, in the
Visual Studio NuGet user interface, check the ""Include prerelease""
box).";
}
string auth =
@"# Authentication
When running on Google Cloud Platform, no action needs to be taken to authenticate.
Otherwise, the simplest way of authenticating your API calls is to
download a service account JSON file then set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to refer to it.
The credentials will automatically be used to authenticate. See the [Getting Started With
Authentication](https://cloud.google.com/docs/authentication/getting-started) guide for more details.";
return(text
.Replace("{{title}}", title)
.Replace("{{description}}", description)
.Replace("{{installation}}", installation)
.Replace("{{auth}}", auth));
}
private static async Task <int> MainAsync(string[] args)
{
var client = new GitHubClient(new ProductHeaderValue(ApplicationName))
{
Credentials = new Octokit.Credentials(args[0])
};
var commit = await FetchRemoteCommitAsync(client);
ValidateLocalRepository(commit);
var apis = ApiMetadata.LoadApis();
var newReleases = await ComputeNewReleasesAsync(client, apis);
ValidateChanges(newReleases);
if (!ConfirmReleases(apis, newReleases))
{
return(0);
}
await MakeReleasesAsync(client, newReleases, commit);
return(0);
}
/// <summary>
/// Extremely crude templating, but just enough for now... it replaces the following tokens:
/// {{title}}: Markdown for the page title with the API ID
/// {{description}}: Markdown for the API description
/// {{installation}}: Markdown for the installation section
/// {{auth}}: Markdown for authentication instructions
/// {{sample:sample ID}}: Include a sample. The sample ID is of the form "Source.Anchor",
/// e.g. "Index.GettingStarted" or "StorageClient.Overview"
/// </summary>
private static string TransformDocTemplate(ApiMetadata api, string text)
{
string title = $"# {api.Id}";
string description = $"`{api.Id}` is a.NET client library for the [{api.ProductName} API]({api.ProductUrl}).";
string version =
$@"Note:
This documentation is for version `{ api.Version}` of the library.
Some samples may not work with other versions.";
string installation =
$@"# Installation
Install the `{api.Id}` package from NuGet. Add it to
your project in the normal way (for example by right-clicking on the
project in Visual Studio and choosing ""Manage NuGet Packages..."").";
if (!api.IsReleaseVersion)
{
installation += $@"
Please ensure you enable pre-release packages (for example, in the
Visual Studio NuGet user interface, check the ""Include prerelease""
box). Some of the following samples might only work with the latest
pre-release version (`{api.Version}`) of `{api.Id}`.";
}
string auth =
@"# Authentication
When running on Google Cloud Platform, no action needs to be taken to authenticate.
Otherwise, the simplest way of authenticating your API calls is to
download a service account JSON file then set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to refer to it.
The credentials will automatically be used to authenticate. See the [Getting Started With
Authentication](https://cloud.google.com/docs/authentication/getting-started) guide for more details.";
var clients = GetClientClasses(api);
string clientClasses = text.Contains("{{client-classes}}") ?
CreateClientClassesDocumentation(api, clients) : "no client classes needed";
var exampleClient = clients.FirstOrDefault();
string clientConstruction =
$@"Create a client instance by calling the static `Create` method. Alternatively,
use the builder class associated with each client class (e.g. {exampleClient}Builder for {exampleClient})
as an easy way of specifying custom credentials, settings, or a custom endpoint.";
string nonProductStub = $@"This package is not a product in its own right; this page is
present as a root for the [API reference documentation](obj/api/{api.Id}.yml)";
text = text
.Replace("{{title}}", title)
.Replace("{{description}}", description)
.Replace("{{version}}", version)
.Replace("{{installation}}", installation)
.Replace("{{auth}}", auth)
.Replace("{{client-classes}}", clientClasses)
.Replace("{{client-construction}}", clientConstruction)
.Replace("{{non-product-stub}}", nonProductStub);
text = Regex.Replace(text, @"\{\{sample:([^\.]+)\.([^}]+)\}\}", "[!code-cs[](obj/snippets/" + api.Id + ".$1.txt#$2)]");
return(text);
}
static int Main(string[] args)
{
var root = DirectoryLayout.DetermineRootDirectory();
var apis = ApiMetadata.LoadApis();
HashSet <string> tags;
using (var repo = new Repository(root))
{
tags = new HashSet <string>(repo.Tags.Select(tag => tag.FriendlyName));
}
var changes = apis.Where(api => !api.Version.EndsWith("00") && !tags.Contains($"{api.Id}-{api.Version}"));
foreach (var api in changes)
{
if (IgnoredApis.Contains(api.Id))
{
Console.WriteLine($"Skipping check for {api.Id} as it doesn't target netstandard2.0");
continue;
}
Console.WriteLine($"Checking compatibility for {api.Id} version {api.Version}");
var prefix = api.Id + "-";
var previousVersions = tags
.Where(tag => tag.StartsWith(prefix))
.Select(tag => tag.Split(new char[] { '-' }, 2)[1])
.Select(v => new StructuredVersion(v))
.OrderBy(v => v)
.ToList();
var newVersion = api.StructuredVersion;
// First perform a "strict" check, where necessary, failing the build if the difference
// is inappropriate.
var(requiredVersion, requiredLevel) = GetRequiredCompatibility(api.StructuredVersion);
if (requiredVersion != null)
{
if (!previousVersions.Contains(requiredVersion))
{
Console.WriteLine($"Expected to check compatibility with {requiredVersion}, but no corresponding tag found");
return(1);
}
var actualLevel = CheckCompatibility(api, requiredVersion);
if (actualLevel < requiredLevel)
{
Console.WriteLine($"Required compatibility level: {requiredLevel}. Actual compatibility level: {actualLevel}.");
return(1);
}
}
// Next log the changes compared with the previous release (if we haven't already diffed it)
// in an informational way. (This can be used to improve or check release notes.)
var lastRelease = previousVersions.LastOrDefault();
if (lastRelease != null && !lastRelease.Equals(requiredVersion))
{
CheckCompatibility(api, lastRelease);
}
}
return(0);
}
static void GenerateSolutionFiles(string apiRoot, ApiMetadata api)
{
var projectDirectories = Directory.GetDirectories(apiRoot)
.Where(pd => Path.GetFileName(pd).StartsWith(api.Id))
.ToList();
HashSet <string> projects = new HashSet <string>();
// We want to include all the project files, and all the project references
// from those project files, being aware that the solution file is already one directory
// higher than the project file...
foreach (var dir in projectDirectories)
{
string projectName = Path.GetFileName(dir);
string projectFile = Path.Combine(dir, $"{projectName}.csproj");
if (File.Exists(projectFile))
{
projects.Add($"{projectName}/{projectName}.csproj");
XDocument doc = XDocument.Load(projectFile);
var projectReferences = doc.Descendants("ProjectReference")
.Select(x => x.Attribute("Include").Value.Replace("\\", "/"))
.Select(x => x.StartsWith("../") ? x.Substring(3) : x);
foreach (var reference in projectReferences)
{
projects.Add(reference);
}
}
}
var solutionFileName = $"{api.Id}.sln";
string fullFile = Path.Combine(apiRoot, solutionFileName);
string beforeHash = GetFileHash(fullFile);
if (!File.Exists(fullFile))
{
Processes.RunDotnet(apiRoot, "new", "sln", "-n", api.Id);
}
else
{
// Optimization: don't run "dotnet sln add" if we can find project entries for all the relevant project
// references already. This is crude, but speeds up the overall process significantly.
var projectLines = File.ReadAllLines(fullFile).Where(line => line.StartsWith("Project(")).ToList();
if (projects.Select(p => $"\"{p.Replace("/", "\\")}\"")
.All(expectedProject => projectLines.Any(pl => pl.Contains(expectedProject))))
{
return;
}
}
// It's much faster to run a single process than to run it once per project.
Processes.RunDotnet(apiRoot, new[] { "sln", solutionFileName, "add" }.Concat(projects).ToArray());
string afterHash = GetFileHash(fullFile);
if (beforeHash != afterHash)
{
Console.WriteLine($"{(beforeHash == null ? "Created" : "Modified")} solution file for {api.Id}");
}
}
private async Task <List <ApiMetadata> > LoadApis()
{
var allContents = await _client.Repository.Content.GetAllContentsByRef(RepositoryOwner, RepositoryName, ApiMetadata.RelativeCatalogPath, _config.Committish);
var json = allContents.Single().Content;
return(ApiMetadata.LoadApisFromJson(json));
}
private static IEnumerable <Release> LoadReleases(Repository repo, ApiMetadata api)
{
var id = api.Id;
var pathPrefix = $"apis/{id}/{id}/";
var projectFile = $"apis/{id}/{id}/{id}.csproj";
// Some versions return forward slashes, some return backslashes :(
Func <string, bool> pathFilter = path => path.Replace('\\', '/').StartsWith(pathPrefix) && path != projectFile;
List <Release> releases = new List <Release>();
StructuredVersion currentVersion = StructuredVersion.FromString(api.Version);
Commit currentTagCommit = null;
// "Pending" as in "haven't been yielded in a release yet"
List <GitCommit> pendingCommits = new List <GitCommit>();
var tagPrefix = $"{id}-";
var versionsCommitId = repo.Tags
.Where(tag => tag.FriendlyName.StartsWith(tagPrefix))
.ToDictionary(tag => tag.Target.Id, tag => tag.FriendlyName.Substring(tagPrefix.Length));
foreach (var commit in repo.Head.Commits)
{
if (CommitContainsApi(commit))
{
pendingCommits.Add(new GitCommit(commit));
}
if (versionsCommitId.TryGetValue(commit.Id, out string version) && !version.StartsWith("0."))
{
yield return(new Release(currentVersion, currentTagCommit, pendingCommits));
// Release constructor clones the list, so we're safe to clear it.
pendingCommits.Clear();
currentTagCommit = commit;
currentVersion = StructuredVersion.FromString(version);
}
}
if (pendingCommits.Count != 0)
{
yield return(new Release(currentVersion, currentTagCommit, pendingCommits));
}
bool CommitContainsApi(Commit commit)
{
if (commit.Parents.Count() != 1)
{
return(false);
}
var tree = commit.Tree;
var parentTree = commit.Parents.First().Tree;
var comparison = repo.Diff.Compare <TreeChanges>(parentTree, tree);
return(comparison.Select(change => change.Path).Any(pathFilter));
}
}
public static void GenerateProjects(string apiRoot, ApiMetadata api, HashSet <string> apiNames)
{
if (api.Type == ApiType.Analyzers)
{
Directory.CreateDirectory(apiRoot);
var mainDirectory = Path.Combine(apiRoot, api.Id);
Directory.CreateDirectory(mainDirectory);
var testDirectory = Path.Combine(apiRoot, api.Id + ".Tests");
Directory.CreateDirectory(testDirectory);
}
// We assume the source directories already exist, either because they've just
// been generated or because they were already there. We infer the type of each
// project based on the directory name. Expected suffixes:
// - None: main API
// - .Snippets: snippets (manual and generated)
// - .Tests: unit tests
// - .IntegrationTests: integration tests
// - .Samples: generated standalone samples
// Anything else will be ignored for now...
var projectDirectories = Directory.GetDirectories(apiRoot)
.Where(pd => Path.GetFileName(pd).StartsWith(api.Id))
.ToList();
foreach (var dir in projectDirectories)
{
string suffix = Path.GetFileName(dir).Substring(api.Id.Length);
switch (suffix)
{
case "":
GenerateMainProject(api, dir, apiNames);
break;
case ".SmokeTests":
GenerateSmokeTestProject(api, dir, apiNames);
break;
case ".IntegrationTests":
case ".Snippets":
case ".Tests":
GenerateTestProject(api, dir, apiNames, isForAnalyzers: api.Type == ApiType.Analyzers);
GenerateCoverageFile(api, dir);
break;
case ".Samples":
GenerateSampleProject(api, dir, apiNames);
break;
}
}
// TODO: Updates for unknown project types? Tricky...
}
private static void IncrementVersion(string[] args)
{
if (args.Length != 1)
{
throw new UserErrorException($"{IncrementVersionCommand} requires one argument: the package ID");
}
string id = args[0];
// It's slightly inefficient that we load the API catalog once here and once later on, and the code duplication
// is annoying too, but it's insignficant really - and at least the code is simple.
var catalog = ApiMetadata.LoadApis();
var api = catalog.FirstOrDefault(x => x.Id == id);
if (api == null)
{
throw new UserErrorException($"API '{id}' not found in API catalog.");
}
var version = IncrementStructuredVersion(api.StructuredVersion).ToString();
SetVersion(id, version);
StructuredVersion IncrementStructuredVersion(StructuredVersion originalVersion)
{
// Any GA version just increments the minor version.
if (originalVersion.Prerelease is null)
{
return(new StructuredVersion(originalVersion.Major, originalVersion.Minor + 1, 0, null));
}
// For prereleases, expect something like "beta01" which should be incremented to "beta02".
var prereleasePattern = new Regex(@"^([^\d]*)(\d+)$");
var match = prereleasePattern.Match(originalVersion.Prerelease);
if (!match.Success)
{
throw new UserErrorException($"Don't know how to auto-increment version '{originalVersion}'");
}
var prefix = match.Groups[1].Value;
var suffix = match.Groups[2].Value;
if (!int.TryParse(suffix, out var counter))
{
throw new UserErrorException($"Don't know how to auto-increment version '{originalVersion}'");
}
counter++;
var newSuffix = counter.ToString().PadLeft(suffix.Length, '0');
return(new StructuredVersion(originalVersion.Major, originalVersion.Minor, originalVersion.Patch, $"{prefix}{newSuffix}"));
}
}
private static string CreateClientClassesDocumentation(ApiMetadata api, List <string> clients)
{
clients = clients.Select(client => $"[{client}](obj/api/{api.Id}.{client}.yml)").ToList(); // Markdown link to API doc
switch (clients.Count)
{
case 0: throw new InvalidOperationException("Couldn't find any clients for {{client-classes}} expansion.");
case 1: return($"All operations are performed through {clients[0]}.");
default:
var list = string.Join("\r\n", clients.Select(client => $"- {client}"));
return($"All operations are performed through the following client classes:\r\n\r\n{list}");
}
}
private static string CreateClientClassesDocumentation(ApiMetadata api, List <string> clients)
{
clients = clients.Select(client => $"[{client}](obj/api/{api.Id}.{client}.yml)").ToList(); // Markdown link to API doc
switch (clients.Count)
{
case 0: return("FIXME"); // No automatic templating for this API
case 1: return($"All operations are performed through {clients[0]}.");
default:
var list = string.Join("\r\n", clients.Select(client => $"- {client}"));
return($"All operations are performed through the following client classes:\r\n\r\n{list}");
}
}
private bool IsGenerated(ApiMetadata api, string googleapisGen)
{
var allLanguagesDirectory = Path.Combine(googleapisGen, api.ProtoPath);
if (!Directory.Exists(allLanguagesDirectory))
{
return(false);
}
var csharpDirectories = Directory.GetDirectories(allLanguagesDirectory, "*-csharp");
return(csharpDirectories.Length == 1 &&
Directory.Exists(Path.Combine(csharpDirectories[0], api.Id)));
}
private static List <ApiVersionPair> FindChangedVersions()
{
var currentCatalog = ApiMetadata.LoadApis();
var masterCatalog = LoadMasterCatalog();
var currentVersions = currentCatalog.ToDictionary(api => api.Id, api => api.Version);
var masterVersions = masterCatalog.ToDictionary(api => api.Id, api => api.Version);
return(currentVersions.Keys.Concat(masterVersions.Keys)
.Distinct()
.OrderBy(id => id)
.Select(id => new ApiVersionPair(id, masterVersions.GetValueOrDefault(id), currentVersions.GetValueOrDefault(id)))
.Where(v => v.NewVersion != v.OldVersion)
.ToList());
}
/// <summary>
/// Extremely crude templating, but just enough for now... it replaces the following tokens:
/// {{title}}: Markdown for the page title with the API ID
/// {{description}}: Markdown for the API description
/// {{installation}}: Markdown for the installation section
/// {{auth}}: Markdown for authentication instructions
/// </summary>
private static string TransformDocTemplate(ApiMetadata api, string text)
{
string title = $"# {api.Id}";
string description = $"`{api.Id}` is a.NET client library for the [{api.ProductName} API]({api.ProductUrl}).";
string version =
$@"Note:
This documentation is for version `{ api.Version}` of the library.
Some samples may not work with other versions.";
string installation =
$@"# Installation
Install the `{api.Id}` package from NuGet. Add it to
your project in the normal way (for example by right-clicking on the
project in Visual Studio and choosing ""Manage NuGet Packages..."").";
if (!api.IsReleaseVersion)
{
installation += $@"
Please ensure you enable pre-release packages (for example, in the
Visual Studio NuGet user interface, check the ""Include prerelease""
box). Some of the following samples might only work with the latest
pre-release version (`{api.Version}`) of `{api.Id}`.";
}
string auth =
@"# Authentication
When running on Google Cloud Platform, no action needs to be taken to authenticate.
Otherwise, the simplest way of authenticating your API calls is to
download a service account JSON file then set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to refer to it.
The credentials will automatically be used to authenticate. See the [Getting Started With
Authentication](https://cloud.google.com/docs/authentication/getting-started) guide for more details.";
string clientClasses = CreateClientClassesDocumentation(api);
string clientConstruction =
@"Create a client instance by calling the static `Create` method, optionally
specifying an end-point or channel and settings.";
return(text
.Replace("{{title}}", title)
.Replace("{{description}}", description)
.Replace("{{version}}", version)
.Replace("{{installation}}", installation)
.Replace("{{auth}}", auth)
.Replace("{{client-classes}}", clientClasses)
.Replace("{{client-construction}}", clientConstruction));
}
// TODO: Find a more robust way of detecting the clients.
private static List <string> GetClientClasses(ApiMetadata api)
{
if (api.Type != ApiType.Grpc)
{
return(new List <string>());
}
var layout = DirectoryLayout.ForApi(api.Id);
var packageSource = Path.Combine(layout.SourceDirectory, api.Id);
var sourceFiles = Directory.GetFiles(packageSource, "*Client.cs").Concat(Directory.GetFiles(packageSource, "*Client.g.cs"));
return(sourceFiles
.Where(file => File.ReadAllText(file).Contains(": gaxgrpc::ServiceSettingsBase")) // Check it contains a generated client
.Select(file => Path.GetFileName(file)) // Just the file name, not full path
.Select(file => file.Split('.')[0]) // Trim .cs or .g.cs
.OrderBy(client => client)
.ToList());
}
private static void GenerateSynthConfiguration(string apiRoot, ApiMetadata api)
{
var synthFile = Path.Combine(apiRoot, "synth.py");
if (api.DetermineAutoGeneratorType() != AutoGeneratorType.Synthtool)
{
// Clean up any previous synth configuration
File.Delete(synthFile);
File.Delete(Path.Combine(apiRoot, "synth.metadata"));
return;
}
// Currently all APIs use the exact same synth file, so we can just replace it every time.
// We may need something more sophisticated in the future.
string content =
@"# GENERATED BY Google.Cloud.Tools.ProjectGenerator - DO NOT EDIT!
import json
import sys
from synthtool import shell
from synthtool import metadata
from pathlib import Path
# generateapis.sh updates synth.metadata itself
metadata.enable_write_metadata(False)
AUTOSYNTH_MULTIPLE_COMMITS = True
# Parent of the script is the API-specific directory
# Parent of the API-specific directory is the apis directory
# Parent of the apis directory is the repo root
root = Path(__file__).parent.parent.parent
package = Path(__file__).parent.name
bash = '/bin/bash'
if sys.platform == 'win32':
bash = 'C:\\Program Files\\Git\\bin\\bash.exe'
shell.run(
(bash, 'generateapis.sh', '--check_compatibility', package),
cwd = root,
hide_output = False)
";
File.WriteAllText(synthFile, content);
}
private static void GenerateSmokeTestProject(ApiMetadata api, string directory, HashSet <string> apiNames)
{
// Don't generate a project file if we've got a placeholder directory
if (Directory.GetFiles(directory, "*.cs", SearchOption.AllDirectories).Length == 0)
{
return;
}
var propertyGroup =
new XElement("PropertyGroup",
new XElement("TargetFramework", "netcoreapp2.1"),
new XElement("OutputType", "Exe"),
new XElement("IsPackable", false));
var dependenciesElement =
new XElement("ItemGroup",
CreateDependencyElement(Path.GetFileName(directory), api.Id, ProjectVersionValue, stableRelease: false, testProject: true, apiNames));
WriteProjectFile(api, directory, propertyGroup, dependenciesElement);
}
