protected override void ExecuteImpl(string[] args)
{
string id = args[0];
var catalog = ApiCatalog.Load();
if (catalog.Apis.Any(api => api.Id == id))
{
throw new UserErrorException($"API {id} already exists in the API catalog.");
}
var root = DirectoryLayout.DetermineRootDirectory();
var googleapis = Path.Combine(root, "googleapis");
var apiIndex = ApiIndex.V1.Index.LoadFromGoogleApis(googleapis);
var targetApi = apiIndex.Apis.FirstOrDefault(api => api.DeriveCSharpNamespace() == id);
if (targetApi is null)
{
var lowerWithoutCloud = id.Replace(".Cloud", "").ToLowerInvariant();
var possibilities = apiIndex.Apis
.Select(api => api.DeriveCSharpNamespace())
.Where(ns => ns.Replace(".Cloud", "").ToLowerInvariant() == lowerWithoutCloud);
throw new UserErrorException(
$"No service found for '{id}'.{Environment.NewLine}Similar possibilities (check options?): {string.Join(", ", possibilities)}");
}
var api = new ApiMetadata
{
Id = id,
ProtoPath = targetApi.Directory,
ProductName = targetApi.Title.EndsWith(" API") ? targetApi.Title[..^ 4] : targetApi.Title,
private static void SetVersion(string id, string version)
{
var catalog = ApiCatalog.Load();
var api = catalog[id];
string oldVersion = api.Version;
api.Version = version;
var layout = DirectoryLayout.ForApi(id);
var apiNames = catalog.CreateIdHashSet();
ProjectGenerator.Program.GenerateMetadataFile(layout.SourceDirectory, api);
ProjectGenerator.Program.GenerateProjects(layout.SourceDirectory, api, apiNames);
ProjectGenerator.Program.RewriteReadme(catalog);
ProjectGenerator.Program.RewriteDocsRootIndex(catalog);
// Update the parsed JObject associated with the ID, and write it back to apis.json.
api.Json["version"] = version;
string formatted = catalog.FormatJson();
File.WriteAllText(ApiCatalog.CatalogPath, formatted);
Console.WriteLine("Updated apis.json");
Console.WriteLine();
Console.WriteLine(new ApiVersionPair(id, oldVersion, version));
}
private static void Execute(string id)
{
var catalog = ApiCatalog.Load();
var api = catalog[id];
if (api.NoVersionHistory)
{
Console.WriteLine($"Skipping version history update for {id}");
return;
}
string historyFilePath = HistoryFile.GetPathForPackage(id);
var root = DirectoryLayout.DetermineRootDirectory();
using (var repo = new Repository(root))
{
var releases = LoadReleases(repo, api).ToList();
if (!File.Exists(historyFilePath))
{
File.WriteAllText(historyFilePath, "# Version history\r\n\r\n");
}
var historyFile = HistoryFile.Load(historyFilePath);
historyFile.MergeReleases(releases);
historyFile.Save(historyFilePath);
}
var relativePath = Path.GetRelativePath(DirectoryLayout.DetermineRootDirectory(), historyFilePath)
.Replace('\\', '/');
Console.WriteLine($"Updated version history file: {relativePath}");
}
protected override void ExecuteImpl(string[] args)
{
var catalog = ApiCatalog.Load();
var root = DirectoryLayout.DetermineRootDirectory();
var googleapis = Path.Combine(root, "googleapis");
var apiIndex = ApiIndex.V1.Index.LoadFromGoogleApis(googleapis);
int modifiedCount = 0;
foreach (var api in catalog.Apis)
{
var indexEntry = apiIndex.Apis.FirstOrDefault(x => x.DeriveCSharpNamespace() == api.Id);
if (indexEntry is null)
{
continue;
}
// Change this line when introducing a new field...
api.Json.Last.AddAfterSelf(new JProperty("serviceConfigFile", indexEntry.ConfigFile));
modifiedCount++;
}
Console.WriteLine($"Modified APIs: {modifiedCount}");
string json = catalog.FormatJson();
// Validate that we can still load it, before saving it to disk...
ApiCatalog.FromJson(json);
File.WriteAllText(ApiCatalog.CatalogPath, json);
}
private static void Execute(string id)
{
var catalog = ApiCatalog.Load();
var api = catalog[id];
if (api.NoVersionHistory)
{
Console.WriteLine($"Skipping version history update for {id}");
return;
}
string historyFilePath = HistoryFile.GetPathForPackage(id);
var root = DirectoryLayout.DetermineRootDirectory();
using var repo = new Repository(root);
var releases = Release.LoadReleases(repo, catalog, api).ToList();
var historyFile = HistoryFile.Load(historyFilePath);
var sectionsInserted = historyFile.MergeReleases(releases);
if (sectionsInserted.Count != 0)
{
historyFile.Save(historyFilePath);
var relativePath = Path.GetRelativePath(DirectoryLayout.DetermineRootDirectory(), historyFilePath)
.Replace('\\', '/');
Console.WriteLine($"Updated version history file: {relativePath}");
Console.WriteLine("New content:");
Console.WriteLine();
foreach (var line in sectionsInserted.SelectMany(section => section.Lines))
{
Console.WriteLine(line);
}
}
}
protected override void ExecuteImpl(string[] args)
{
string configFile = args[0];
var json = File.ReadAllText(configFile);
var config = JsonConvert.DeserializeObject <BatchReleaseConfig>(json);
var criteria = config.GetCriteria().ToList();
if (criteria.Count != 1)
{
throw new UserErrorException("Batch release config must specify exactly one criterion.");
}
if (!config.DryRun)
{
var root = DirectoryLayout.DetermineRootDirectory();
using var repo = new Repository(root);
if (repo.RetrieveStatus().IsDirty)
{
throw new UserErrorException("In non-dry-run mode, the current branch must not have changes.");
}
}
var catalog = ApiCatalog.Load();
var criterion = criteria[0];
var proposals = criterion.GetProposals(catalog);
foreach (var proposal in proposals)
{
// Note: This takes into account the dry-run flag.
proposal.Execute(config);
}
}
private static int MainImpl(string[] args)
{
if (args.Length != 1)
{
throw new UserErrorException("Please specify the API name");
}
string api = args[0];
var layout = DirectoryLayout.ForApi(api);
var apiMetadata = ApiCatalog.Load()[api];
string output = layout.DocsOutputDirectory;
if (Directory.Exists(output))
{
Directory.Delete(output, true);
}
Directory.CreateDirectory(output);
var apiDirectory = layout.SourceDirectory;
var projects = Project.LoadProjects(apiDirectory).ToList();
CreateDocfxJson(api, projects, output);
CopyAndGenerateArticles(apiMetadata, layout.DocsSourceDirectory, output);
CreateToc(api, output);
return(0);
}
protected override void ExecuteImpl(string[] args)
{
ValidateCommonHiddenProductionDependencies();
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
ValidateApiCatalog(catalog);
Console.WriteLine($"API catalog contains {catalog.Apis.Count} entries");
// Now we know we can parse the API catalog, let's reformat it.
ReformatApiCatalog(catalog);
RewriteReadme(catalog);
RewriteRenovate(catalog);
HashSet <string> apiNames = catalog.CreateIdHashSet();
foreach (var api in catalog.Apis)
{
var path = Path.Combine(root, "apis", api.Id);
GenerateProjects(path, api, apiNames);
GenerateSolutionFiles(path, api);
GenerateDocumentationStub(path, api);
GenerateSynthConfiguration(path, api);
GenerateOwlBotConfiguration(path, api);
GenerateMetadataFile(path, api);
}
}
protected override void ExecuteImpl(string[] args)
{
string id = args[0];
var catalog = ApiCatalog.Load();
var api = catalog[id];
Console.WriteLine($"Current version of {id} in the API catalog: {api.Version}");
}
private Program(string apiId)
{
DirectoryLayout layout = DirectoryLayout.ForApi(apiId);
_apiId = apiId;
_outputRoot = layout.DocsOutputDirectory;
_devSiteRoot = Path.Combine(_outputRoot, "devsite");
_apiCatalog = ApiCatalog.Load();
_apiIds = _apiCatalog.CreateIdHashSet();
}
public void Execute(string[] args)
{
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
HashSet <string> tags;
using (var repo = new Repository(root))
{
tags = new HashSet <string>(repo.Tags.Select(tag => tag.FriendlyName));
}
List <ApiMetadata> apisToCheck = args.Length == 0
? catalog.Apis.Where(api => !api.Version.EndsWith("00") && !tags.Contains($"{api.Id}-{api.Version}")).ToList()
// Note: this basically validates the command line arguments.
: args.Select(arg => catalog[arg]).ToList();
foreach (var api in apisToCheck)
{
Console.WriteLine($"Checking compatibility for {api.Id} version {api.Version}");
var prefix = api.Id + "-";
var lastVersion = tags
.Where(tag => tag.StartsWith(prefix))
.Select(tag => tag.Split(new char[] { '-' }, 2)[1])
.Where(v => !v.StartsWith("0")) // We can reasonably ignore old 0.x versions
.Select(StructuredVersion.FromString)
.OrderBy(v => v)
.LastOrDefault();
if (lastVersion is null)
{
Console.WriteLine("No previous versions released; ignoring.");
continue;
}
var newVersion = api.StructuredVersion;
// If we're releasing a new version, we should check against the previous one.
// (For example, if this PR creates 1.2.0, then check against 1.1.0.)
// Otherwise, just expect minor changes.
Level requiredLevel = Level.Minor;
if (!lastVersion.Equals(newVersion))
{
requiredLevel =
lastVersion.Major != newVersion.Major ? Level.Major // Major version bump: anything goes
: lastVersion.Minor != newVersion.Minor ? Level.Minor // Minor version bump: minor changes are okay
: Level.Identical; // Patch version bump: API should be identical
}
var actualLevel = CheckCompatibility(api, lastVersion);
if (actualLevel < requiredLevel)
{
throw new UserErrorException($"Required compatibility level: {requiredLevel}. Actual compatibility level: {actualLevel}.");
}
}
}
protected override void ExecuteImpl(string[] args)
{
var catalog = ApiCatalog.Load();
var lagging = catalog.Apis.Where(api => api.CanHaveGaRelease && api.StructuredVersion.Prerelease is string);
Console.WriteLine($"Lagging packages:");
foreach (var api in lagging)
{
Console.WriteLine($"{api.Id} ({api.Version})");
}
}
protected override void ExecuteImpl(string[] args)
{
string googleapisGen = args[0];
ApiCatalog catalog = ApiCatalog.Load();
var apis = catalog.Apis.Where(x => x.DetermineAutoGeneratorType() == AutoGeneratorType.OwlBot).ToList();
var ungenerated = apis.Where(api => !IsGenerated(api, googleapisGen)).ToList();
foreach (var api in ungenerated)
{
Console.WriteLine($"Ungenerated API: {api.Id}");
}
Console.WriteLine($"Total OwlBot APIs: {apis.Count}; ungenerated APIs: {ungenerated.Count}");
}
protected List <ApiVersionPair> FindChangedVersions()
{
var currentCatalog = ApiCatalog.Load();
var primaryCatalog = LoadPrimaryCatalog();
var currentVersions = currentCatalog.CreateRawVersionMap();
var primaryVersions = primaryCatalog.CreateRawVersionMap();
return(currentVersions.Keys.Concat(primaryVersions.Keys)
.Distinct()
.OrderBy(id => id)
.Select(id => new ApiVersionPair(id, primaryVersions.GetValueOrDefault(id), currentVersions.GetValueOrDefault(id)))
.Where(v => v.NewVersion != v.OldVersion)
.ToList());
}
protected override void ExecuteImpl(string[] args)
{
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
using (var repo = new Repository(root))
{
var allTags = repo.Tags.OrderByDescending(GitHelpers.GetDate).ToList();
foreach (var api in catalog.Apis)
{
MaybeShowStale(repo, allTags, api);
}
}
}
private static List <ApiVersionPair> FindChangedVersions()
{
var currentCatalog = ApiCatalog.Load();
var masterCatalog = LoadMasterCatalog();
var currentVersions = currentCatalog.CreateRawVersionMap();
var masterVersions = masterCatalog.CreateRawVersionMap();
return(currentVersions.Keys.Concat(masterVersions.Keys)
.Distinct()
.OrderBy(id => id)
.Select(id => new ApiVersionPair(id, masterVersions.GetValueOrDefault(id), currentVersions.GetValueOrDefault(id)))
.Where(v => v.NewVersion != v.OldVersion)
.ToList());
}
private void CreateDocUploaderMetadata()
{
var apiMetadata = ApiCatalog.Load()[_apiId];
var docUploaderMetadata = new JObject
{
["version"] = apiMetadata.Version,
["language"] = "dotnet",
["name"] = apiMetadata.Id,
["xrefs"] = new JArray(GetXrefUrls(apiMetadata)),
["xrefServices"] = new JArray("https://xref.docs.microsoft.com/query?uid={uid}")
};
string json = docUploaderMetadata.ToString(Formatting.Indented);
File.WriteAllText(Path.Combine(_devSiteRoot, "docs.metadata.json"), json);
}
protected override void ExecuteImpl(string[] args)
{
Console.WriteLine($"Lagging packages (package ID, current version, date range of current version prerelease series):");
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
using (var repo = new Repository(root))
{
var allTags = repo.Tags.OrderByDescending(GitHelpers.GetDate).ToList();
foreach (var api in catalog.Apis)
{
MaybeShowLagging(repo, allTags, api);
}
}
}
private static void IncrementVersion(string[] args)
{
if (args.Length != 1)
{
throw new UserErrorException($"{IncrementVersionCommand} requires one argument: the package ID");
}
string id = args[0];
// It's slightly inefficient that we load the API catalog once here and once later on, and the code duplication
// is annoying too, but it's insignficant really - and at least the code is simple.
var catalog = ApiCatalog.Load();
var api = catalog[id];
var version = IncrementStructuredVersion(api.StructuredVersion).ToString();
SetVersion(id, version);
StructuredVersion IncrementStructuredVersion(StructuredVersion originalVersion)
{
// Any GA version just increments the minor version.
if (originalVersion.Prerelease is null)
{
return(new StructuredVersion(originalVersion.Major, originalVersion.Minor + 1, 0, null));
}
// For prereleases, expect something like "beta01" which should be incremented to "beta02".
var prereleasePattern = new Regex(@"^([^\d]*)(\d+)$");
var match = prereleasePattern.Match(originalVersion.Prerelease);
if (!match.Success)
{
throw new UserErrorException($"Don't know how to auto-increment version '{originalVersion}'");
}
var prefix = match.Groups[1].Value;
var suffix = match.Groups[2].Value;
if (!int.TryParse(suffix, out var counter))
{
throw new UserErrorException($"Don't know how to auto-increment version '{originalVersion}'");
}
counter++;
var newSuffix = counter.ToString().PadLeft(suffix.Length, '0');
return(new StructuredVersion(originalVersion.Major, originalVersion.Minor, originalVersion.Patch, $"{prefix}{newSuffix}"));
}
}
private void ValidateProjectReferences()
{
var catalog = ApiCatalog.Load();
var newReleaseIds = FindChangedVersions().Select(change => change.Id).ToList();
foreach (var id in newReleaseIds)
{
var api = catalog[id];
var projectReferences = api.Dependencies.Where(p => p.Value == "project").Select(p => p.Key);
var badReferences = projectReferences.Except(newReleaseIds).ToList();
if (badReferences.Any())
{
throw new UserErrorException(
$"Project {api.Id} contains project references to projects outside the release set: {string.Join(", ", badReferences)}");
}
}
}
protected override void ExecuteImpl(string[] args)
{
var catalog = ApiCatalog.Load();
var apiNames = catalog.CreateIdHashSet();
foreach (var api in catalog.Apis)
{
GenerateProjectsCommand.UpdateDependencies(catalog, api);
var layout = DirectoryLayout.ForApi(api.Id);
GenerateProjectsCommand.GenerateMetadataFile(layout.SourceDirectory, api);
GenerateProjectsCommand.GenerateProjects(layout.SourceDirectory, api, apiNames);
}
string formatted = catalog.FormatJson();
File.WriteAllText(ApiCatalog.CatalogPath, formatted);
Console.WriteLine("Updated apis.json");
}
private static void Main()
{
var directory = ServiceDirectory.LoadFromGoogleapis();
var catalog = ApiCatalog.Load();
var paths = new HashSet <string>(catalog.Apis.Select(api => api.ProtoPath));
var serviceDirectories = directory.Services
.Where(service => service.Stable)
.Select(service => service.ServiceDirectory).ToList();
var ungenerated = serviceDirectories
.Except(paths)
.Except(catalog.IgnoredPaths)
.OrderBy(path => path).ToList();
ungenerated.ForEach(Console.WriteLine);
}
static void Main(string[] args)
{
bool attentionOnly = args.Contains("--attention");
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
using (var repo = new Repository(root))
{
var diff = repo.Diff;
var tags = repo.Tags;
var shaToTimestamp = repo.Commits.ToDictionary(commit => commit.Sha, commit => commit.Committer.When);
foreach (var api in catalog.Apis)
{
DisplayApi(attentionOnly, api, tags, shaToTimestamp);
}
}
}
protected override void ExecuteImpl(string[] args)
{
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
using var repo = new Repository(root);
var pendingChangesByApi = GitHelpers.GetPendingChangesByApi(repo, catalog);
// TODO: Work out what to do with unreleased APIs. Do we want to show them or not?
foreach (var api in catalog.Apis)
{
var release = pendingChangesByApi[api];
var commits = release.Commits;
if (commits.Count == 0)
{
continue;
}
string changes = commits.Count == 1 ? "change" : "changes";
Console.WriteLine($"{api.Id,-50} {release.ReleaseDate:yyyy-MM-dd}: {release.Version} + {commits.Count} {changes}");
}
}
public void Execute(string[] args)
{
var root = DirectoryLayout.DetermineRootDirectory();
var catalog = ApiCatalog.Load();
HashSet <string> tags;
using (var repo = new Repository(root))
{
tags = new HashSet <string>(repo.Tags.Select(tag => tag.FriendlyName));
}
List <ApiMetadata> apisToCheck = args.Length == 0
? catalog.Apis.Where(api => !api.Version.EndsWith("00") && !tags.Contains($"{api.Id}-{api.Version}")).ToList()
// Note: this basically validates the command line arguments.
: args.Select(arg => catalog[arg]).ToList();
foreach (var api in apisToCheck)
{
Console.WriteLine($"Checking compatibility for {api.Id} version {api.Version}");
var prefix = api.Id + "-";
var taggedVersions = tags
.Where(tag => tag.StartsWith(prefix))
.Select(tag => tag.Split(new char[] { '-' }, 2)[1])
.Where(v => !v.StartsWith("0")) // We can reasonably ignore old 0.x versions
.Select(StructuredVersion.FromString)
.OrderBy(v => v)
.ToList();
var comparisons = GetComparisons(api.StructuredVersion, taggedVersions);
foreach (var(oldVersion, requiredLevel) in comparisons)
{
var actualLevel = CheckCompatibility(api, oldVersion);
if (actualLevel < requiredLevel)
{
throw new UserErrorException($"Required compatibility level: {requiredLevel}. Actual compatibility level: {actualLevel}.");
}
}
}
}
protected override void ExecuteImpl(string[] args)
{
var catalog = ApiCatalog.Load();
// Note: for now, we could actually load it from apis/ServiceDirectory, but hey...
var directory = ServiceDirectory.LoadFromGoogleapis();
var stabilityFilter = BuildStabilityFilter(args[0]);
var ignoredOrGeneratedPaths = new HashSet <string>(catalog.IgnoredPaths.Keys);
ignoredOrGeneratedPaths.UnionWith(catalog.Apis.Select(api => api.ProtoPath));
var missingServices = directory.Services
.Where(stabilityFilter)
.Where(svc => !ignoredOrGeneratedPaths.Contains(svc.ServiceDirectory))
.ToList();
Console.WriteLine($"Missing services: {missingServices.Count}");
foreach (var service in missingServices)
{
Console.WriteLine(service.ServiceDirectory);
}
}
protected override void ExecuteImpl(string[] args)
{
var catalog = ApiCatalog.Load();
var root = DirectoryLayout.DetermineRootDirectory();
var googleapis = Path.Combine(root, "googleapis");
var apiIndex = ApiIndex.V1.Index.LoadFromGoogleApis(googleapis);
var stabilityFilter = BuildStabilityFilter(args[0]);
var ignoredOrGeneratedPaths = new HashSet <string>(catalog.IgnoredPaths.Keys);
ignoredOrGeneratedPaths.UnionWith(catalog.Apis.Select(api => api.ProtoPath));
var missingApis = apiIndex.Apis
.Where(stabilityFilter)
.Where(api => !ignoredOrGeneratedPaths.Contains(api.Directory))
.ToList();
Console.WriteLine($"Missing apis: {missingApis.Count}");
foreach (var api in missingApis)
{
Console.WriteLine($"{api.Directory} => {api.DeriveCSharpNamespace()}");
}
}