public static async Task <NetworkCredential> ResetWindowsUserAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
string username,
TimeSpan timeout,
CancellationToken token)
{
using (var timeoutCts = new CancellationTokenSource())
{
timeoutCts.CancelAfter(timeout);
using (var combinedCts = CancellationTokenSource.CreateLinkedTokenSource(timeoutCts.Token, token))
{
try
{
return(await ResetWindowsUserAsync(
resource,
instanceRef,
username,
combinedCts.Token).ConfigureAwait(false));
}
catch (Exception e) when(e.IsCancellation() && timeoutCts.IsCancellationRequested)
{
TraceSources.Common.TraceError(e);
// This task was cancelled because of a timeout, not because
// the enclosing job was cancelled.
throw new PasswordResetException(
$"Timeout waiting for Compute Engine agent to reset password for user {username}. " +
"Verify that the agent is running.");
}
}
}
}
/// <summary>
/// Read serial port output as a continuous stream.
/// </summary>
public static IAsyncReader <string> GetSerialPortOutputStream(
this InstancesResource resource,
InstanceLocator instanceRef,
ushort port)
{
return(new SerialPortStream(resource, instanceRef, port));
}
public void SetUp()
{
var service = TestProject.CreateComputeService();
this.instancesResource = service.Instances;
this.projectsResource = service.Projects;
}
/// <summary>
/// Read serial port output as a continuous stream.
/// </summary>
public static SerialPortStream GetSerialPortOutputStream(
this InstancesResource resource,
VmInstanceReference instanceRef,
ushort port)
{
return(new SerialPortStream(resource, instanceRef, port));
}
public SerialPortStream(
InstancesResource instancesResource,
InstanceLocator instanceRef,
ushort port)
{
this.instancesResource = instancesResource;
this.port = port;
this.instance = instanceRef;
}
public SerialPortStream(
InstancesResource instancesResource,
VmInstanceReference instanceRef,
ushort port)
{
this.instancesResource = instancesResource;
this.port = port;
this.instance = instanceRef;
}
/// <summary>
/// Read serial port output as a continuous stream.
/// </summary>
public static SerialPortStream GetSerialPortOutputStream(
this InstancesResource resource,
string project,
string zone,
string instance,
ushort port)
{
return(GetSerialPortOutputStream(
resource,
new VmInstanceReference(project, zone, instance),
port));
}
/// <summary>
/// Adds or overwrites a metadata key/value pair to a GCE
/// instance. Any existing metadata is kept as is.
/// </summary>
public static Task AddMetadataAsync(
this InstancesResource resource,
string project,
string zone,
string instance,
string key,
string value)
{
return(AddMetadataAsync(
resource,
new VmInstanceReference(project, zone, instance),
key,
value));
}
/// <summary>
/// Reset a SAM account password. If the SAM account does not exist,
/// it is created and made a local Administrator.
/// </summary>
/// <see href="https://cloud.google.com/compute/docs/instances/windows/automate-pw-generation"/>
public static Task <NetworkCredential> ResetWindowsUserAsync(
this InstancesResource resource,
string project,
string zone,
string instance,
string username,
CancellationToken token)
{
return(ResetWindowsUserAsync(
resource,
new InstanceLocator(project, zone, instance),
username,
token));
}
public static Task AddMetadataAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
Metadata metadata,
CancellationToken token)
{
return(UpdateMetadataAsync(
resource,
instanceRef,
existingMetadata =>
{
existingMetadata.Add(metadata);
},
token));
}
private static async Task AwaitInstanceCreatedAndReady(
InstancesResource resource,
InstanceLocator locator)
{
for (int i = 0; i < 60; i++)
{
try
{
var instance = await resource.Get(
locator.ProjectId,
locator.Zone,
locator.Name)
.ExecuteAsync();
// Determine the name of the guest attribute we need to await.
var guestAttributeToAwait = instance.Metadata.Items
.EnsureNotNull()
.FirstOrDefault(item => item.Key == GuestAttributeToAwaitKey)
.Value;
var request = resource.GetGuestAttributes(
locator.ProjectId,
locator.Zone,
locator.Name);
request.QueryPath = GuestAttributeNamespace + "/";
var guestAttributes = await request.ExecuteAsync();
if (guestAttributes
.QueryValue
.Items
.Where(item => item.Namespace__ == GuestAttributeNamespace &&
item.Key == guestAttributeToAwait)
.Any())
{
return;
}
}
catch (Exception)
{ }
TraceSources.Common.TraceVerbose(
"Waiting for instance {0} to become ready...", locator.Name);
await Task.Delay(5 * 1000);
}
throw new TimeoutException($"Timeout waiting for {locator} to become ready");
}
/// <summary>
/// Adds or overwrites a metadata key/value pair to a GCE
/// instance. Any existing metadata is kept as is.
/// </summary>
public static Task AddMetadataAsync(
this InstancesResource resource,
string project,
string zone,
string instance,
string key,
string value,
CancellationToken token)
{
return(AddMetadataAsync(
resource,
new InstanceLocator(project, zone, instance),
key,
value,
token));
}
/// <summary>
/// Query a metadata entry for an instance.
/// </summary>
/// <returns>null if not set/found</returns>
public static async Task <Metadata.ItemsData> QueryMetadataKeyAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
string key)
{
var instance = await resource.Get(
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name).ExecuteAsync().ConfigureAwait(false);
if (instance.Metadata == null || instance.Metadata.Items == null)
{
return(null);
}
return(instance.Metadata.Items
.Where(i => i.Key == key)
.FirstOrDefault());
}
//---------------------------------------------------------------------
// Instance metadata.
//---------------------------------------------------------------------
/// <summary>
/// Modify instance metadata.
/// </summary>
public static async Task UpdateMetadataAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
Action <Metadata> updateMetadata,
CancellationToken token,
uint maxAttempts = DefaultAttempts)
{
using (CommonTraceSources.Default.TraceMethod().WithParameters(instanceRef))
{
await UpdateMetadataAsync(
async() =>
{
var instance = await resource.Get(
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name)
.ExecuteAsync(token)
.ConfigureAwait(false);
return(instance.Metadata);
},
async metadata =>
{
await resource.SetMetadata(
metadata,
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name)
.ExecuteAndAwaitOperationAsync(
instanceRef.ProjectId,
token)
.ConfigureAwait(false);
},
updateMetadata,
maxAttempts)
.ConfigureAwait(false);
}
}
/// <summary>
/// Adds or overwrites a metadata key/value pair to a GCE
/// instance. Any existing metadata is kept as is.
/// </summary>
public static Task AddMetadataAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
string key,
string value,
CancellationToken token)
{
return(AddMetadataAsync(
resource,
instanceRef,
new Metadata()
{
Items = new List <Metadata.ItemsData>()
{
new Metadata.ItemsData()
{
Key = key,
Value = value
}
}
},
token));
}
public void SetUp()
{
this.instancesResource = TestProject.CreateComputeService().Instances;
}
/// <summary>
/// Adds or overwrites a metadata key/value pair to a GCE
/// instance. Any existing metadata is kept as is.
/// </summary>
public static async Task AddMetadataAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
string key,
string value,
CancellationToken token)
{
using (TraceSources.Common.TraceMethod().WithParameters(instanceRef, key))
{
for (int attempt = 0; attempt < 6; attempt++)
{
var instance = await resource.Get(
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name).ExecuteAsync(token).ConfigureAwait(false);
var metadata = instance.Metadata;
if (metadata.Items == null)
{
metadata.Items = new List <Metadata.ItemsData>();
}
var existingEntry = metadata.Items
.Where(i => i.Key == key)
.FirstOrDefault();
if (existingEntry != null)
{
existingEntry.Value = value;
}
else
{
metadata.Items.Add(new Metadata.ItemsData()
{
Key = key,
Value = value
});
}
TraceSources.Common.TraceVerbose("Setting metdata {0} on {1}...", key, instanceRef.Name);
try
{
await resource.SetMetadata(
metadata,
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name).ExecuteAndAwaitOperationAsync(instanceRef.ProjectId, token).ConfigureAwait(false);
break;
}
catch (GoogleApiException e)
{
if (e.Error != null && e.Error.Code == 412)
{
// Fingerprint mismatch - that happens when somebody else updated metadata
// in patallel.
int backoff = 100;
TraceSources.Common.TraceWarning(
"SetMetadata failed with {0} - retrying after {1}ms", e.Message,
e.Error?.Code,
backoff);
await Task.Delay(backoff).ConfigureAwait(false);
}
else
{
TraceSources.Common.TraceWarning(
"Setting metdata failed {0} (code error {1})", e.Message,
e.Error?.Code);
throw;
}
}
}
}
}
/// <summary>
/// Reset a SAM account password. If the SAM account does not exist,
/// it is created and made a local Administrator.
/// </summary>
/// <see href="https://cloud.google.com/compute/docs/instances/windows/automate-pw-generation"/>
public static async Task <NetworkCredential> ResetWindowsUserAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
string username,
CancellationToken token)
{
using (TraceSources.Common.TraceMethod().WithParameters(instanceRef, username))
using (RSACryptoServiceProvider rsa = new RSACryptoServiceProvider(RsaKeySize))
{
var keyParameters = rsa.ExportParameters(false);
var requestPayload = new RequestPayload()
{
ExpireOn = DateTime.UtcNow.AddMinutes(5),
Username = username,
Email = username,
Modulus = Convert.ToBase64String(keyParameters.Modulus),
Exponent = Convert.ToBase64String(keyParameters.Exponent),
};
// Send the request to the instance via a special metadata entry.
try
{
var requestJson = JsonConvert.SerializeObject(requestPayload);
await resource.AddMetadataAsync(
instanceRef,
MetadataKey,
requestJson,
token).ConfigureAwait(false);
}
catch (GoogleApiException e) when(e.Error != null && e.Error.Code == 404)
{
TraceSources.Common.TraceVerbose("Instance does not exist: {0}", e.Message);
throw new PasswordResetException(
$"Instance {instanceRef.Name} was not found.");
}
catch (GoogleApiException e) when(e.Error == null || e.Error.Code == 403)
{
TraceSources.Common.TraceVerbose(
"Setting request payload metadata failed: {0} ({1})",
e.Message,
e.Error?.Errors.EnsureNotNull().Select(er => er.Reason).FirstOrDefault());
// Setting metadata failed due to lack of permissions. Note that
// the Error object is not always populated, hence the OR filter.
throw new PasswordResetException(
"You do not have sufficient permissions to reset a Windows password. " +
"You need the 'Service Account User' and " +
"'Compute Instance Admin' roles (or equivalent custom roles) " +
"to perform this action.");
}
// Read response from serial port.
using (var serialPortStream = resource.GetSerialPortOutputStream(
instanceRef,
SerialPort))
{
// It is rare, but sometimes a single JSON can be split over multiple
// API reads. Therefore, maintain a buffer.
var logBuffer = new StringBuilder(64 * 1024);
while (true)
{
TraceSources.Common.TraceVerbose("Waiting for agent to supply response...");
token.ThrowIfCancellationRequested();
string logDelta = await serialPortStream.ReadAsync(token).ConfigureAwait(false);
if (string.IsNullOrEmpty(logDelta))
{
// Reached end of stream, wait and try again.
await Task.Delay(500, token).ConfigureAwait(false);
continue;
}
logBuffer.Append(logDelta);
var response = logBuffer.ToString().Split('\n')
.Where(line => line.Contains(requestPayload.Modulus))
.FirstOrDefault();
if (response == null)
{
// That was not the output we are looking for, keep reading.
continue;
}
var responsePayload = JsonConvert.DeserializeObject <ResponsePayload>(response);
if (!string.IsNullOrEmpty(responsePayload.ErrorMessage))
{
throw new PasswordResetException(responsePayload.ErrorMessage);
}
var password = rsa.Decrypt(
Convert.FromBase64String(responsePayload.EncryptedPassword),
true);
return(new NetworkCredential(
username,
new UTF8Encoding().GetString(password),
null));
}
}
}
}
/// <summary>Constructs a new resource.</summary>
public LocationsResource(Google.Apis.Services.IClientService service)
{
this.service = service;
Instances = new InstancesResource(service);
}
public void SetUp()
{
this.instancesResource = ComputeEngine.Connect().Service.Instances;
}
/// <summary>
/// Adds or overwrites a metadata key/value pair to a GCE
/// instance. Any existing metadata is kept as is.
/// </summary>
public static async Task AddMetadataAsync(
this InstancesResource resource,
VmInstanceReference instanceRef,
string key,
string value)
{
var instance = await resource.Get(
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.InstanceName).ExecuteAsync().ConfigureAwait(false);
var metadata = instance.Metadata;
if (metadata.Items == null)
{
metadata.Items = new List <Metadata.ItemsData>();
}
var existingEntry = metadata.Items
.Where(i => i.Key == key)
.FirstOrDefault();
if (existingEntry != null)
{
existingEntry.Value = value;
}
else
{
metadata.Items.Add(new Metadata.ItemsData()
{
Key = key,
Value = value
});
}
TraceSources.Compute.TraceVerbose("Setting metdata {0} on {1}...", key, instanceRef.InstanceName);
await resource.SetMetadata(
metadata,
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.InstanceName).ExecuteAsync().ConfigureAwait(false);
// Setting the metadata might fail silently, cf b/140226028 - so check if
// it worked. Sometimes, the change also takes a few seconds to apply (sic).
for (int attempt = 0; attempt < 10; attempt++)
{
var appliedValue = await resource.QueryMetadataKeyAsync(
instanceRef,
key).ConfigureAwait(false);
if (appliedValue != null && appliedValue.Value == value)
{
// Success.
return;
}
else
{
// Wait and retry.
TraceSources.Compute.TraceVerbose("Metdata change not applied yet, trying again...");
await Task.Delay(500).ConfigureAwait(false);
}
}
throw new GoogleApiException("Compute", "Setting metdata failed");
}
/// <summary>
/// Adds or overwrites a metadata key/value pair to a GCE
/// instance. Any existing metadata is kept as is.
/// </summary>
public static async Task AddMetadataAsync(
this InstancesResource resource,
InstanceLocator instanceRef,
Metadata metadata,
CancellationToken token)
{
using (TraceSources.Common.TraceMethod().WithParameters(instanceRef))
{
for (int attempt = 0; attempt < 6; attempt++)
{
TraceSources.Common.TraceVerbose("Adding metadata {0} on {1}...", metadata, instanceRef.Name);
//
// NB. Metadata must be updated all-at-once. Therefore,
// fetch the existing entries first before merging them
// with the new entries.
//
var instance = await resource.Get(
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name).ExecuteAsync(token).ConfigureAwait(false);
var mergedMetadata = instance.Metadata;
mergedMetadata.Add(metadata);
try
{
await resource.SetMetadata(
mergedMetadata,
instanceRef.ProjectId,
instanceRef.Zone,
instanceRef.Name).ExecuteAndAwaitOperationAsync(instanceRef.ProjectId, token).ConfigureAwait(false);
break;
}
catch (GoogleApiException e)
{
if (e.Error != null && e.Error.Code == 412)
{
// Fingerprint mismatch - that happens when somebody else updated metadata
// in patallel.
int backoff = 100;
TraceSources.Common.TraceWarning(
"SetMetadata failed with {0} - retrying after {1}ms", e.Message,
e.Error?.Code,
backoff);
await Task.Delay(backoff).ConfigureAwait(false);
}
else
{
TraceSources.Common.TraceWarning(
"Setting metdata failed {0} (code error {1})", e.Message,
e.Error?.Code);
throw;
}
}
}
}
}
public async Task AddExistingInstances(
InstancesResource instancesResource,
DisksResource disksResource,
string projectId)
{
// NB. Instances.list returns the disks associated with each
// instance, but lacks the information about the source image.
// Therefore, load disks separately.
using (TraceSources.LogAnalysis.TraceMethod().WithParameters(projectId))
{
// TODO: Paging
var disks = await disksResource.AggregatedList(projectId).ExecuteAsync();
var sourceImagaesByDisk = disks.Items == null
? new Dictionary <string, string>()
: disks.Items.Values
.Where(v => v.Disks != null)
.EnsureNotNull()
.SelectMany(v => v.Disks)
.EnsureNotNull()
.ToDictionary(d => d.SelfLink, d => d.SourceImage);
// TODO: Paging
var instances = await instancesResource.AggregatedList(projectId).ExecuteAsync();
if (instances.Items != null)
{
foreach (var list in instances.Items.Values)
{
if (list.Instances == null)
{
continue;
}
foreach (var instance in list.Instances)
{
TraceSources.LogAnalysis.TraceVerbose("Adding {0}", instance.Id);
var bootDiskUrl = instance.Disks
.EnsureNotNull()
.Where(d => d.Boot != null && d.Boot.Value)
.EnsureNotNull()
.Select(d => d.Source)
.EnsureNotNull()
.FirstOrDefault();
GlobalResourceReference image = null;
if (bootDiskUrl != null &&
sourceImagaesByDisk.TryGetValue(bootDiskUrl, out string imageUrl) &&
imageUrl != null)
{
image = GlobalResourceReference.FromString(imageUrl);
}
AddExistingInstance(
(ulong)instance.Id.Value,
new VmInstanceReference(
projectId,
ShortZoneIdFromUrl(instance.Zone),
instance.Name),
image,
instance.Status == "RUNNING"
? InstanceState.Running
: InstanceState.Terminated,
DateTime.Now,
instance.Scheduling.NodeAffinities != null && instance.Scheduling.NodeAffinities.Any()
? Tenancy.SoleTenant
: Tenancy.Fleet);
}
}
}
}
}
