public async Task RemoveActivation(ClusterIdentity clusterIdentity, PID pid, CancellationToken ct)
{
Logger.LogDebug("Removing activation: {ClusterIdentity} {@PID}", clusterIdentity, pid);
var key = GetKey(clusterIdentity);
await _asyncSemaphore.WaitAsync(() => _pids.DeleteManyAsync(p => p.Key == key && p.UniqueIdentity == pid.Id, ct));
}
public async Task<SpawnLock?> TryAcquireLock(ClusterIdentity clusterIdentity, CancellationToken ct)
{
var requestId = Guid.NewGuid().ToString("N");
var hasLock = await _asyncSemaphore.WaitAsync(() => TryAcquireLockAsync(clusterIdentity, requestId)).ConfigureAwait(false);
return hasLock ? new SpawnLock(requestId, clusterIdentity) : null;
}
public async Task Setup()
{
var echoProps = Props.FromFunc(ctx => {
if (ctx.Sender is not null)
{
ctx.Respond(ctx.Message !);
}
return(Task.CompletedTask);
}
);
if (RequestDeduplication)
{
echoProps = echoProps.WithClusterRequestDeduplication(TimeSpan.FromSeconds(30));
}
var echoKind = new ClusterKind(Kind, echoProps);
if (LocalAffinity)
{
echoKind.WithLocalAffinityRelocationStrategy();
}
var sys = new ActorSystem(new ActorSystemConfig())
.WithRemote(GrpcNetRemoteConfig.BindToLocalhost(9090))
.WithCluster(ClusterConfig().WithClusterKind(echoKind));
_cluster = sys.Cluster();
await _cluster.StartMemberAsync();
_id = ClusterIdentity.Create("1", Kind);
await _cluster.RequestAsync <int>(_id.Identity, _id.Kind, 1, CancellationToken.None);
}
public async Task PurgesPidCacheOnNullResponse()
{
var system = new ActorSystem();
system.Metrics.Register(new ClusterMetrics(system.Metrics));
var props = Props.FromProducer(() => new EchoActor());
var deadPid = system.Root.SpawnNamed(props, "stopped");
var alivePid = system.Root.SpawnNamed(props, "alive");
await system.Root.StopAsync(deadPid).ConfigureAwait(false);
var dummyIdentityLookup = new DummyIdentityLookup(alivePid);
var pidCache = new PidCache();
var logger = Log.CreateLogger("dummylog");
var clusterIdentity = new ClusterIdentity {
Identity = "identity", Kind = "kind"
};
pidCache.TryAdd(clusterIdentity, deadPid);
var requestAsyncStrategy = new DefaultClusterContext(dummyIdentityLookup, pidCache, new ClusterContextConfig(), system.Shutdown);
var res = await requestAsyncStrategy.RequestAsync <Pong>(clusterIdentity, new Ping { Message = "msg" }, system.Root,
new CancellationTokenSource(6000).Token
);
res !.Message.Should().Be("msg");
var foundInCache = pidCache.TryGet(clusterIdentity, out var pidInCache);
foundInCache.Should().BeTrue();
pidInCache.Should().BeEquivalentTo(alivePid);
}
public async Task<StoredActivation?> TryGetExistingActivation(
ClusterIdentity clusterIdentity,
CancellationToken ct
)
{
var activationStatus = await LookupKey(GetDb(), IdKey(clusterIdentity)).ConfigureAwait(false);
return activationStatus?.Activation;
}
public async Task <SpawnLock?> TryAcquireLock(
ClusterIdentity clusterIdentity,
CancellationToken ct
)
{
var requestId = Guid.NewGuid().ToString();
var hasLock = await TryAcquireLockAsync(clusterIdentity, requestId, ct);
return(hasLock ? new SpawnLock(requestId, clusterIdentity) : null);
}
public Task RemovePidAsync(ClusterIdentity clusterIdentity, PID pid, CancellationToken ct)
{
var activationTerminated = new ActivationTerminated
{
Pid = pid,
ClusterIdentity = clusterIdentity,
};
_cluster.MemberList.BroadcastEvent(activationTerminated);
return(Task.CompletedTask);
}
public async Task <StoredActivation?> TryGetExistingActivation(
ClusterIdentity clusterIdentity,
CancellationToken ct
)
{
var pidLookup = await LookupKey(GetKey(clusterIdentity), ct);
return(pidLookup?.Address == null || pidLookup?.UniqueIdentity == null
? null
: new StoredActivation(pidLookup.MemberId !,
PID.FromAddress(pidLookup.Address, pidLookup.UniqueIdentity)
));
}
public AzureHDInsightClusterIdentity(ClusterIdentity clusterIdentity)
{
PrincipalId = clusterIdentity?.PrincipalId;
TenantId = clusterIdentity?.TenantId;
Type = clusterIdentity?.Type;
UserAssignedIdentities = clusterIdentity?.UserAssignedIdentities != null ? new Dictionary <string, AzureHDInsightUserAssignedIdentity>() : null;
if (UserAssignedIdentities != null)
{
foreach (var entry in clusterIdentity.UserAssignedIdentities)
{
UserAssignedIdentities.Add(entry.Key, new AzureHDInsightUserAssignedIdentity(entry.Value));
}
}
}
public void CannotStoreWithoutLock()
{
var timeout = new CancellationTokenSource(1000).Token;
var activator = GetFakeActivator();
var identity = new ClusterIdentity {
Kind = "thing", Identity = NextId().ToString()
};
var spawnLock = new SpawnLock("not-a-lock", identity);
var pid = Activate(activator, identity);
_storage.Invoking(storage =>
storage.StoreActivation(activator.Id, spawnLock, pid, timeout)
).Should().Throw <LockNotFoundException>();
}
internal Action <MessageEnvelope> ForActor(ClusterIdentity identity, PID activation)
{
var activeRemotes = new BitArray(16);
return(envelope => {
if (envelope.Message is Stopped)
{
Invalidate(identity, activation, activeRemotes);
}
else if (IsRemote(envelope.Sender))
{
AddRemote(envelope.Sender !, activeRemotes);
}
});
}
private Task<bool> TryAcquireLockAsync(
ClusterIdentity clusterIdentity,
string requestId
)
{
var key = IdKey(clusterIdentity);
var db = GetDb();
var transaction = db.CreateTransaction();
transaction.AddCondition(Condition.KeyNotExists(key));
transaction.HashSetAsync(key, LockId, requestId);
transaction.KeyExpireAsync(key, _maxLockTime);
return transaction.ExecuteAsync();
}
public async Task <StoredActivation?> WaitForActivation(
ClusterIdentity clusterIdentity,
CancellationToken ct
)
{
var key = GetKey(clusterIdentity);
var pidLookupEntity = await LookupKey(key, ct);
var lockId = pidLookupEntity?.LockedBy;
if (lockId != null)
{
//There is an active lock on the pid, spin wait
var i = 0;
do
{
await Task.Delay(20 *i, ct);
}while ((pidLookupEntity = await LookupKey(key, ct))?.LockedBy == lockId && ++i < 10);
}
//the lookup entity was lost, stale lock maybe?
if (pidLookupEntity == null)
{
return(null);
}
//lookup was unlocked, return this pid
if (pidLookupEntity.LockedBy == null)
{
return(new StoredActivation(pidLookupEntity.MemberId !,
PID.FromAddress(pidLookupEntity.Address !, pidLookupEntity.UniqueIdentity !)
));
}
//Still locked but not by the same request that originally locked it, so not stale
if (pidLookupEntity.LockedBy != lockId)
{
return(null);
}
//Stale lock. just delete it and let cluster retry
// _logger.LogDebug($"Stale lock: {pidLookupEntity.Key}");
await RemoveLock(new SpawnLock(lockId !, clusterIdentity), CancellationToken.None);
return(null);
}
public async Task GlobalLockActivatesOnceOnly()
{
var timeout = new CancellationTokenSource(1000).Token;
var identity = new ClusterIdentity {
Kind = "thing", Identity = NextId().ToString()
};
const int attempts = 10;
var locks = await Task.WhenAll(Enumerable.Range(1, attempts)
.Select(i => _storage.TryAcquireLock(identity, timeout))
);
var successFullLock = locks.Where(it => it != null).ToList();
successFullLock.Should().HaveCount(1);
successFullLock.Single() !.ClusterIdentity.Should().BeEquivalentTo(identity);
}
public async Task RemovesLockIfStale()
{
var timeout = new CancellationTokenSource(10000).Token;
var identity = new ClusterIdentity {
Kind = "thing", Identity = NextId().ToString()
};
await _storage.TryAcquireLock(identity, timeout);
var activation = await _storage.WaitForActivation(identity, timeout);
var spawnLock = await _storage.TryAcquireLock(identity, timeout);
activation.Should().BeNull("We did not activate it");
spawnLock.Should().NotBeNull(
"When an activation did not occur, the storage implementation should discard the lock"
);
}
private void Invalidate(ClusterIdentity identity, PID activation, BitArray activeRemotes)
{
var message = new ActivationTerminated
{
ClusterIdentity = identity,
Pid = activation
};
var remotesToInvalidate = Cluster.MemberList.GetAllMembers()
.Select(m => Cluster.MemberList.GetMetaMember(m.Id))
.Where(m => activeRemotes.Length > m.Index && activeRemotes[m.Index])
.Select(m => m.Member.Address);
foreach (var address in remotesToInvalidate)
{
Cluster.System.Root.Send(PID.FromAddress(address, ActorName), message);
}
}
public async Task CanDeleteSpawnLocks()
{
var timeout = new CancellationTokenSource(1000).Token;
var identity = new ClusterIdentity {
Kind = "thing", Identity = NextId().ToString()
};
var spawnLock = await _storage.TryAcquireLock(identity, timeout);
spawnLock.Should().NotBeNull();
await _storage.RemoveLock(spawnLock !, timeout);
var secondLock = await _storage.TryAcquireLock(identity, timeout);
secondLock.Should().NotBeNull("The initial lock should be cleared, and a second lock can be acquired.");
}
public async Task <PID?> GetAsync(ClusterIdentity clusterIdentity, CancellationToken ct)
{
ct = CancellationTokens.WithTimeout(_getPidTimeout);
//Get address to node owning this ID
var identityOwner = _partitionManager.Selector.GetIdentityOwner(clusterIdentity.Identity);
Logger.LogDebug("Identity belongs to {address}", identityOwner);
if (string.IsNullOrEmpty(identityOwner))
{
return(null);
}
var remotePid = PartitionManager.RemotePartitionIdentityActor(identityOwner);
var req = new ActivationRequest
{
ClusterIdentity = clusterIdentity
};
Logger.LogDebug("Requesting remote PID from {Partition}:{Remote} {@Request}", identityOwner, remotePid, req
);
try
{
var resp = await _cluster.System.Root.RequestAsync <ActivationResponse>(remotePid, req, ct);
return(resp.Pid);
}
//TODO: decide if we throw or return null
catch (DeadLetterException)
{
Logger.LogInformation("Remote PID request deadletter {@Request}, identity Owner {Owner}", req, identityOwner);
return(null);
}
catch (TimeoutException)
{
Logger.LogInformation("Remote PID request timeout {@Request}, identity Owner {Owner}", req, identityOwner);
return(null);
}
catch (Exception e)
{
Logger.LogError(e, "Error occured requesting remote PID {@Request}, identity Owner {Owner}", req, identityOwner);
return(null);
}
}
public Task RemoveActivation(ClusterIdentity clusterIdentity, PID pid, CancellationToken ct)
{
Logger.LogDebug("Removing activation: {ClusterIdentity} {@PID}", clusterIdentity, pid);
const string removePid = "local pidEntry = redis.call('HMGET', KEYS[1], 'pid', 'adr', 'mid');\n" +
"if pidEntry[1]~=ARGV[1] or pidEntry[2]~=ARGV[2] then return 0 end;\n" + // id / address matches
"local memberKey = ARGV[3] .. pidEntry[3];\n" +
"redis.call('SREM', memberKey, KEYS[1] .. '');" +
"return redis.call('DEL', KEYS[1]);";
var key = IdKey(clusterIdentity);
return _asyncSemaphore.WaitAsync(()
=> {
return GetDb().ScriptEvaluateAsync(removePid, new[] {key}, new RedisValue[] {pid.Id, pid.Address, _memberKey.ToString()}
);
}
);
}
private async Task <bool> TryAcquireLockAsync(
ClusterIdentity clusterIdentity,
string requestId,
CancellationToken ct
)
{
var key = GetKey(clusterIdentity);
var lockEntity = new PidLookupEntity
{
Address = null,
Identity = clusterIdentity.Identity,
Key = key,
Kind = clusterIdentity.Kind,
LockedBy = requestId,
Revision = 1,
MemberId = null
};
try
{
//be 100% sure own the lock here
await _asyncSemaphore.WaitAsync(() => _pids.InsertOneAsync(lockEntity, new InsertOneOptions(), ct));
Logger.LogDebug("Got lock on first try for {ClusterIdentity}", clusterIdentity);
return(true);
}
catch (MongoWriteException)
{
var l = await _asyncSemaphore.WaitAsync(() => _pids.ReplaceOneAsync(x => x.Key == key && x.LockedBy == null && x.Revision == 0,
lockEntity,
new ReplaceOptions
{
IsUpsert = false
}, ct
)
);
//if l.MatchCount == 1, then one document was updated by us, and we should own the lock, no?
var gotLock = l.IsAcknowledged && l.ModifiedCount == 1;
Logger.LogDebug("Did {Got}get lock on second try for {ClusterIdentity}", gotLock ? "" : "not ", clusterIdentity);
return(gotLock);
}
}
/// <summary>
/// Create cluster create parameters for ADLS Gen2 relevant tests
/// </summary>
/// <param name="commonData"></param>
/// <param name="storageAccountName"></param>
/// <param name="storageResourceId"></param>
/// <param name="msiResourceId"></param>
/// <param name="createParams"></param>
/// <returns></returns>
public static ClusterCreateParametersExtended PrepareClusterCreateParamsForADLSv2(
this CommonTestFixture commonData,
string storageAccountName,
string storageResourceId,
string msiResourceId,
ClusterCreateParametersExtended createParams = null)
{
var createParamsForADLSv2 = createParams ?? commonData.PrepareClusterCreateParams();
bool isDefault = !createParamsForADLSv2.Properties.StorageProfile.Storageaccounts.Any();
createParamsForADLSv2.Properties.StorageProfile.Storageaccounts.Add(
new HDInsightStorageAccount
{
Name = storageAccountName + commonData.DfsEndpointSuffix,
IsDefault = isDefault,
FileSystem = commonData.ContainerName.ToLowerInvariant(),
ResourceId = storageResourceId,
MsiResourceId = msiResourceId
}
);
var identity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>
{
{ msiResourceId, new ClusterIdentityUserAssignedIdentitiesValue() }
}
};
if (createParamsForADLSv2.Identity == null)
{
createParamsForADLSv2.Identity = identity;
}
else
{
// At this point, only user-assigned managed identity is supported by HDInsight.
// So identity type is not checked.
createParamsForADLSv2.Identity.UserAssignedIdentities.Union(identity.UserAssignedIdentities);
}
return(createParamsForADLSv2);
}
public async Task CanWaitForActivation()
{
var activator = GetFakeActivator();
var timeout = CancellationTokens.WithTimeout(15 * 1000);
var identity = new ClusterIdentity {
Kind = "thing", Identity = NextId().ToString()
};
var spawnLock = await _storage.TryAcquireLock(identity, timeout);
var pid = Activate(activator, identity);
_ = SafeTask.Run(async() => {
await Task.Delay(500, timeout);
await _storage.StoreActivation(activator.Id, spawnLock !, pid, timeout);
}, timeout
);
var activation = await _storage.WaitForActivation(identity, timeout);
activation.Should().NotBeNull();
activation !.MemberId.Should().Be(activator.Id);
activation !.Pid.Should().BeEquivalentTo(pid);
}
public async Task GlobalLockActivatesOnceOnlyAcrossMultipleClients()
{
var timeout = new CancellationTokenSource(1000).Token;
var identity = new ClusterIdentity {
Kind = "thing", Identity = "1234"
};
const int attempts = 10;
var locks = await Task.WhenAll(Enumerable.Range(1, attempts)
.SelectMany(_ => new[]
{
_storage.TryAcquireLock(identity, timeout),
_storageInstance2.TryAcquireLock(identity, timeout)
}
)
);
var successfulLock = locks.Where(it => it != null).ToList();
successfulLock.Should().HaveCount(1);
successfulLock.Single() !.ClusterIdentity.Should().BeEquivalentTo(identity);
}
public async Task CannotTakeLockWhenAlreadyActivated()
{
var activator = GetFakeActivator();
var timeout = new CancellationTokenSource(1000).Token;
var identity = new ClusterIdentity {
Kind = "thing", Identity = NextId().ToString()
};
var spawnLock = await _storage.TryAcquireLock(identity, timeout);
var pid = Activate(activator, identity);
await _storage.StoreActivation(activator.Id, spawnLock !, pid, timeout);
var activation = await _storage.TryGetExistingActivation(identity, timeout);
activation.Should().NotBeNull();
activation !.MemberId.Should().Be(activator.Id);
activation !.Pid.Should().BeEquivalentTo(pid);
var noLock = await _storage.TryAcquireLock(identity, timeout);
noLock.Should().BeNull("Since the activation is active, it should not be possible to take the lock");
}
public async Task<StoredActivation?> WaitForActivation(
ClusterIdentity clusterIdentity,
CancellationToken ct
)
{
var timer = Stopwatch.StartNew();
var key = IdKey(clusterIdentity);
var db = GetDb();
var activationStatus = await LookupKey(db, key).ConfigureAwait(false);
var lockId = activationStatus?.ActiveLockId;
if (lockId != null)
{
//There is an active lock on the pid, spin wait
var i = 1;
do await Task.Delay(20 * i++, ct);
while (!ct.IsCancellationRequested
&& _maxLockTime > timer.Elapsed
&& (activationStatus = await LookupKey(db, key).ConfigureAwait(false))?.ActiveLockId == lockId
);
}
//the lookup entity was lost, stale lock maybe?
if (activationStatus == null) return null;
//lookup was unlocked, return this pid
if (activationStatus.Activation != null)
return activationStatus.Activation;
//Still locked but not by the same request that originally locked it, so not stale
if (activationStatus.ActiveLockId != lockId) return null;
//Stale lock. just delete it and let cluster retry
await RemoveLock(new SpawnLock(lockId!, clusterIdentity), CancellationToken.None).ConfigureAwait(false);
return null;
}
public void DumpState(ClusterIdentity clusterIdentity)
{
Console.WriteLine("Memberlist members:");
_cluster.MemberList.DumpState();
Console.WriteLine("Partition manager selector:");
_partitionManager.Selector.DumpState();
//Get address to node owning this ID
var identityOwner = _partitionManager.Selector.GetIdentityOwner(clusterIdentity.Identity);
Console.WriteLine("Identity owner for ID:");
Console.WriteLine(identityOwner);
var remotePid = PartitionManager.RemotePartitionIdentityActor(identityOwner);
var req = new ActivationRequest
{
ClusterIdentity = clusterIdentity
};
var resp = _cluster.System.Root.RequestAsync <ActivationResponse>(remotePid, req, CancellationTokens.WithTimeout(5000)).Result;
Console.WriteLine("Target Pid:");
if (resp == null)
{
Console.WriteLine("Null response");
}
else if (resp.Pid == null)
{
Console.WriteLine("Null PID");
}
else
{
Console.WriteLine(resp.Pid);
}
}
public void CanCreateNewHDInsightCluster_Disk_Encryption()
{
string AssignedIdentity = "/subscriptions/00000000-aaaa-bbbb-cccc-dddddddddddd/resourcegroups/group-unittest/providers/microsoft.managedidentity/userassignedidentities/ami-unittest";
string EncryptionVaultUri = "https://vault-unittest.vault.azure.net:443";
string EncryptionKeyVersion = "00000000000000000000000000000000";
string EncryptionKeyName = "key-unittest";
string EncryptionAlgorithm = "RSA-OAEP";
string sparkClusterType = "Spark";
cmdlet.ClusterName = ClusterName;
cmdlet.ResourceGroupName = ResourceGroupName;
cmdlet.ClusterSizeInNodes = ClusterSize;
cmdlet.Location = Location;
cmdlet.HttpCredential = _httpCred;
cmdlet.DefaultStorageAccountName = StorageName;
cmdlet.DefaultStorageAccountKey = StorageKey;
cmdlet.ClusterType = "Spark";
cmdlet.SshCredential = _sshCred;
cmdlet.EncryptionAlgorithm = EncryptionAlgorithm;
cmdlet.EncryptionKeyName = EncryptionKeyName;
cmdlet.EncryptionKeyVersion = EncryptionKeyVersion;
cmdlet.EncryptionVaultUri = EncryptionVaultUri;
cmdlet.AssignedIdentity = AssignedIdentity;
var ClusterIdentity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>
{
{
AssignedIdentity, new ClusterIdentityUserAssignedIdentitiesValue(principalId: "PrincipalId", clientId: "ClientId")
}
}
};
var cluster = new Cluster(id: "id", name: ClusterName, identity: ClusterIdentity)
{
Location = Location,
Properties = new ClusterGetProperties
{
ClusterVersion = "3.6",
ClusterState = "Running",
ClusterDefinition = new ClusterDefinition
{
Kind = sparkClusterType
},
QuotaInfo = new QuotaInfo
{
CoresUsed = 24
},
OsType = OSType.Linux,
DiskEncryptionProperties = new DiskEncryptionProperties()
{
KeyName = EncryptionKeyName,
KeyVersion = EncryptionKeyVersion,
VaultUri = EncryptionVaultUri,
EncryptionAlgorithm = EncryptionAlgorithm,
MsiResourceId = AssignedIdentity
},
},
};
var coreConfigs = new Dictionary <string, string>
{
{ "fs.defaultFS", "wasb://giyertestcsmv2@" + StorageName },
{
"fs.azure.account.key." + StorageName,
StorageKey
}
};
var gatewayConfigs = new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", _httpCred.UserName },
{ "restAuthCredential.password", _httpCred.Password.ConvertToString() }
};
var configurations = new Dictionary <string, Dictionary <string, string> >
{
{ "core-site", coreConfigs },
{ "gateway", gatewayConfigs }
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, OSType.Linux, It.Is <ClusterCreateParameters>(
parameters =>
parameters.ClusterSizeInNodes == ClusterSize &&
parameters.DefaultStorageInfo as AzureStorageInfo != null &&
((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountName == StorageName &&
((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountKey == StorageKey &&
parameters.Location == Location &&
parameters.UserName == _httpCred.UserName &&
parameters.Password == _httpCred.Password.ConvertToString() &&
parameters.ClusterType == sparkClusterType &&
parameters.SshUserName == _sshCred.UserName &&
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
parameters.DiskEncryptionProperties.VaultUri == EncryptionVaultUri &&
parameters.DiskEncryptionProperties.KeyName == EncryptionKeyName &&
parameters.DiskEncryptionProperties.KeyVersion == EncryptionKeyVersion &&
parameters.DiskEncryptionProperties.EncryptionAlgorithm == EncryptionAlgorithm &&
parameters.DiskEncryptionProperties.MsiResourceId == AssignedIdentity), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool?>()))
.Returns(cluster)
.Verifiable();
cmdlet.ExecuteCmdlet();
commandRuntimeMock.VerifyAll();
commandRuntimeMock.Verify(f => f.WriteObject(It.Is <AzureHDInsightCluster>(
clusterout =>
clusterout.ClusterState == "Running" &&
clusterout.ClusterType == sparkClusterType &&
clusterout.ClusterVersion == "3.6" &&
clusterout.CoresUsed == 24 &&
clusterout.Location == Location &&
clusterout.Name == ClusterName &&
clusterout.OperatingSystemType == OSType.Linux &&
clusterout.DiskEncryption.KeyName == EncryptionKeyName &&
clusterout.DiskEncryption.KeyVersion == EncryptionKeyVersion &&
clusterout.DiskEncryption.VaultUri == EncryptionVaultUri &&
clusterout.DiskEncryption.EncryptionAlgorithm == EncryptionAlgorithm &&
clusterout.DiskEncryption.MsiResourceId == AssignedIdentity &&
clusterout.AssignedIdentity.Type == ResourceIdentityType.UserAssigned &&
clusterout.AssignedIdentity.UserAssignedIdentities[AssignedIdentity].ClientId == "ClientId" &&
clusterout.AssignedIdentity.UserAssignedIdentities[AssignedIdentity].PrincipalId == "PrincipalId"
)), Times.Once);
}
public override void ExecuteCmdlet()
{
foreach (var component in ComponentVersion.Where(component => !clusterComponentVersion.ContainsKey(component.Key)))
{
clusterComponentVersion.Add(component.Key, component.Value);
}
// Construct Configurations
foreach (var config in Configurations.Where(config => !clusterConfigurations.ContainsKey(config.Key)))
{
clusterConfigurations.Add(config.Key, config.Value);
}
// Add cluster username/password to gateway config.
ClusterCreateHelper.AddClusterCredentialToGatewayConfig(HttpCredential, clusterConfigurations);
// Construct OS Profile
OsProfile osProfile = ClusterCreateHelper.CreateOsProfile(SshCredential, SshPublicKey);
// Construct Virtual Network Profile
VirtualNetworkProfile vnetProfile = ClusterCreateHelper.CreateVirtualNetworkProfile(VirtualNetworkId, SubnetName);
// Handle storage account
StorageProfile storageProfile = new StorageProfile()
{
Storageaccounts = new List <StorageAccount> {
}
};
if (StorageAccountType == null || StorageAccountType == StorageType.AzureStorage)
{
var azureStorageAccount = ClusterCreateHelper.CreateAzureStorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageContainer, this.DefaultContext.Environment.StorageEndpointSuffix);
storageProfile.Storageaccounts.Add(azureStorageAccount);
}
else if (StorageAccountType == StorageType.AzureDataLakeStore)
{
ClusterCreateHelper.AddAzureDataLakeStorageGen1ToCoreConfig(StorageAccountResourceId, StorageRootPath, this.DefaultContext.Environment.AzureDataLakeStoreFileSystemEndpointSuffix, clusterConfigurations);
}
else if (StorageAccountType == StorageType.AzureDataLakeStorageGen2)
{
var adlsgen2Account = ClusterCreateHelper.CreateAdlsGen2StorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageFileSystem, StorageAccountManagedIdentity, this.DefaultContext.Environment.StorageEndpointSuffix);
storageProfile.Storageaccounts.Add(adlsgen2Account);
}
// Handle additional storage accounts
foreach (
var storageAccount in
AdditionalStorageAccounts.Where(
storageAccount => !clusterAdditionalStorageAccounts.ContainsKey(storageAccount.Key)))
{
clusterAdditionalStorageAccounts.Add(storageAccount.Key, storageAccount.Value);
}
ClusterCreateHelper.AddAdditionalStorageAccountsToCoreConfig(clusterAdditionalStorageAccounts, clusterConfigurations);
// Handle script action
foreach (var action in ScriptActions.Where(action => clusterScriptActions.ContainsKey(action.Key)))
{
clusterScriptActions.Add(action.Key,
action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
}
// Handle metastore
if (OozieMetastore != null)
{
ClusterCreateHelper.AddOozieMetastoreToConfigurations(OozieMetastore, clusterConfigurations);
}
if (HiveMetastore != null)
{
ClusterCreateHelper.AddHiveMetastoreToConfigurations(HiveMetastore, clusterConfigurations);
}
// Handle ADLSGen1 identity
if (!string.IsNullOrEmpty(CertificatePassword))
{
if (!string.IsNullOrEmpty(CertificateFilePath))
{
CertificateFileContents = File.ReadAllBytes(CertificateFilePath);
}
ClusterCreateHelper.AddDataLakeStorageGen1IdentityToIdentityConfig(
GetApplicationId(ApplicationId), GetTenantId(AadTenantId), CertificateFileContents, CertificatePassword, clusterConfigurations,
this.DefaultContext.Environment.ActiveDirectoryAuthority, this.DefaultContext.Environment.DataLakeEndpointResourceId);
}
// Handle Kafka Rest Proxy
KafkaRestProperties kafkaRestProperties = null;
if (KafkaClientGroupId != null && KafkaClientGroupName != null)
{
kafkaRestProperties = new KafkaRestProperties()
{
ClientGroupInfo = new ClientGroupInfo(KafkaClientGroupName, KafkaClientGroupId)
};
}
// Compute profile contains headnode, workernode, zookeepernode, edgenode, kafkamanagementnode, idbrokernode, etc.
ComputeProfile computeProfile = ClusterCreateHelper.CreateComputeProfile(osProfile, vnetProfile, clusterScriptActions, ClusterType, ClusterSizeInNodes, HeadNodeSize, WorkerNodeSize, ZookeeperNodeSize, EdgeNodeSize, KafkaManagementNodeSize, EnableIDBroker.IsPresent);
// Handle SecurityProfile
SecurityProfile securityProfile = ClusterCreateHelper.ConvertAzureHDInsightSecurityProfileToSecurityProfile(SecurityProfile, AssignedIdentity);
// Handle DisksPerWorkerNode feature
Role workerNode = Utils.ExtractRole(ClusterNodeType.WorkerNode.ToString(), computeProfile);
if (DisksPerWorkerNode > 0)
{
workerNode.DataDisksGroups = new List <DataDisksGroups>()
{
new DataDisksGroups()
{
DisksPerNode = DisksPerWorkerNode
}
};
}
// Handle ClusterIdentity
ClusterIdentity clusterIdentity = null;
if (AssignedIdentity != null || StorageAccountManagedIdentity != null)
{
clusterIdentity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>()
};
if (AssignedIdentity != null)
{
clusterIdentity.UserAssignedIdentities.Add(AssignedIdentity, new ClusterIdentityUserAssignedIdentitiesValue());
}
if (StorageAccountManagedIdentity != null)
{
clusterIdentity.UserAssignedIdentities.Add(StorageAccountManagedIdentity, new ClusterIdentityUserAssignedIdentitiesValue());
}
}
// Handle CMK feature
DiskEncryptionProperties diskEncryptionProperties = null;
if (EncryptionKeyName != null && EncryptionKeyVersion != null && EncryptionVaultUri != null)
{
diskEncryptionProperties = new DiskEncryptionProperties()
{
KeyName = EncryptionKeyName,
KeyVersion = EncryptionKeyVersion,
VaultUri = EncryptionVaultUri,
EncryptionAlgorithm = EncryptionAlgorithm != null ? EncryptionAlgorithm : JsonWebKeyEncryptionAlgorithm.RSAOAEP,
MsiResourceId = AssignedIdentity
};
}
// Handle encryption at host feature
if (EncryptionAtHost != null)
{
if (diskEncryptionProperties != null)
{
diskEncryptionProperties.EncryptionAtHost = EncryptionAtHost;
}
else
{
diskEncryptionProperties = new DiskEncryptionProperties()
{
EncryptionAtHost = EncryptionAtHost
};
}
}
// Handle autoscale featurer
Autoscale autoscaleParameter = null;
if (AutoscaleConfiguration != null)
{
autoscaleParameter = AutoscaleConfiguration.ToAutoscale();
workerNode.AutoscaleConfiguration = autoscaleParameter;
}
// Construct cluster create parameter
ClusterCreateParametersExtended createParams = new ClusterCreateParametersExtended
{
Location = Location,
//Tags = Tags, //To Do add this Tags parameter
Properties = new ClusterCreateProperties
{
Tier = ClusterTier,
ClusterDefinition = new ClusterDefinition
{
Kind = ClusterType ?? "Hadoop",
ComponentVersion = clusterComponentVersion,
Configurations = clusterConfigurations
},
ClusterVersion = Version ?? "default",
KafkaRestProperties = kafkaRestProperties,
ComputeProfile = computeProfile,
OsType = OSType,
SecurityProfile = securityProfile,
StorageProfile = storageProfile,
DiskEncryptionProperties = diskEncryptionProperties,
//handle Encryption In Transit feature
EncryptionInTransitProperties = EncryptionInTransit != null ? new EncryptionInTransitProperties()
{
IsEncryptionInTransitEnabled = EncryptionInTransit
} : null,
MinSupportedTlsVersion = MinSupportedTlsVersion
},
Identity = clusterIdentity
};
var cluster = HDInsightManagementClient.CreateCluster(ResourceGroupName, ClusterName, createParams);
if (cluster != null)
{
WriteObject(new AzureHDInsightCluster(cluster));
}
}
private string GetKey(ClusterIdentity clusterIdentity) => $"{_clusterName}/{clusterIdentity}";
private RedisKey IdKey(ClusterIdentity clusterIdentity) => _clusterIdentityKey.Append(clusterIdentity.ToString());