//--// public bool GetInputs( out AzurePrepInputs result ) { result = new AzurePrepInputs( ); result.Credentials = AzureConsoleHelper.GetUserSubscriptionCredentials( ); if( result.Credentials == null ) { result = null; return false; } for( ;; ) { Console.WriteLine( "Enter a name for Service Bus Namespace (only letters and digits, less than 17 chars long)." ); Console.WriteLine( "(Note that fully qualified path may also be subject to further length restrictions.)" ); result.NamePrefix = Console.ReadLine( ); if( string.IsNullOrEmpty( result.NamePrefix ) || !CheckNamePrefix( result.NamePrefix ) ) { Console.WriteLine( "Namespace prefix should contain only letters and digits and have length less than 17." ); continue; } if( ConsoleHelper.Confirm( "Are you sure you want to create a namespace called " + result.NamePrefix + "?" ) ) { break; } } if( string.IsNullOrEmpty( result.NamePrefix ) ) { result = null; return false; } result.Location = SelectRegion( result ); result.SBNamespace = result.NamePrefix + "-ns"; result.StorageAccountName = result.NamePrefix.ToLowerInvariant( ) + "storage"; result.EventHubNameDevices = "ehdevices"; result.EventHubNameAlerts = "ehalerts"; return true; }
private void CreateStreamAnalyticsJob( string nameSuffix, string query, string resourceGroupName, AzurePrepInputs azurePrepIn, EventHubDescription ehInput, EventHubDescription ehOutput ) { const string inputName = "DevicesInput"; const string outputName = "output"; string jobName = azurePrepIn.NamePrefix + nameSuffix; string transformationName = jobName + "-tr"; var computeClient = new StreamAnalyticsManagementClient( azurePrepIn.Credentials ); var serialization = new JsonSerialization { Type = "JSON", Properties = new JsonSerializationProperties { Encoding = "UTF8" } }; List<Input> jobInputs = new List<Input> { new Input { Name = inputName, Properties = new StreamInputProperties { DataSource = new EventHubStreamInputDataSource { Properties = new EventHubStreamInputDataSourceProperties { EventHubName = ehInput.Path, ServiceBusNamespace = azurePrepIn.SBNamespace, SharedAccessPolicyName = "StreamingAnalytics", SharedAccessPolicyKey = ( ehInput.Authorization.First( ( d ) => String.Equals( d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase) ) as SharedAccessAuthorizationRule ).PrimaryKey, } }, Serialization = serialization } } }; List<Output> jobOutputs = new List<Output> { new Output { Name = outputName, Properties = new OutputProperties { DataSource = new EventHubOutputDataSource { Properties = new EventHubOutputDataSourceProperties { EventHubName = ehOutput.Path, ServiceBusNamespace = azurePrepIn.SBNamespace, SharedAccessPolicyName = "StreamingAnalytics", SharedAccessPolicyKey = ( ehOutput.Authorization.First( ( d ) => String.Equals( d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase) ) as SharedAccessAuthorizationRule ).PrimaryKey, } }, Serialization = serialization } } }; bool created = true; try { var jobCreateResponse = computeClient.StreamingJobs.CreateOrUpdateAsync( resourceGroupName, new JobCreateOrUpdateParameters { Job = new Job { Name = jobName, Location = azurePrepIn.Location, Properties = new JobProperties { Sku = new Sku { //should be "standart" according to https://msdn.microsoft.com/en-us/library/azure/dn834994.aspx Name = "standard" }, EventsOutOfOrderPolicy = "drop", EventsOutOfOrderMaxDelayInSeconds = 10, Inputs = jobInputs, Outputs = jobOutputs, Transformation = new Transformation { Name = transformationName, Properties = new TransformationProperties { Query = query, StreamingUnits = 1 } } } } } ).Result; } catch( Exception ex ) { _ConsoleBuffer.Add( "Exception on creation Stream Analytics Job " + jobName + ": " + ex.Message ); _ConsoleBuffer.Add( "Inner exception message: " + ex.InnerException.Message ); created = false; } if( created ) { _ConsoleBuffer.Add( "Stream Analytics job " + jobName + " created." ); } }
private void CreateStreamAnalyticsJobs( AzurePrepInputs azurePrepIn, AzurePrepOutputs azurePrepOut ) { string resourceGroupName = SelectResourceGroup( azurePrepIn ); string path = Path.GetDirectoryName( System.Reflection.Assembly.GetEntryAssembly( ).Location ); path += "\\..\\..\\..\\..\\StreamAnalyticsQueries"; foreach( string filename in Directory.GetFiles( path ) ) { string extension = Path.GetExtension( filename ); if( extension != null && extension.Contains( "sql" ) ) { string nameWithoutExtension = Path.GetFileNameWithoutExtension( filename ); EventHubDescription ehOutput = ( filename.ToLower( ).Contains( "aggregates" ) || azurePrepOut.ehAlerts == null ) ? azurePrepOut.ehDevices : azurePrepOut.ehAlerts; if( ehOutput == null ) { _ConsoleBuffer.Add( string.Format( " Skip creating {0} Stream Analytics job because there is no output Event Hub...", nameWithoutExtension ) ); continue; } string queryFilename = filename; ConsoleHelper.AskAndPerformAction( "Do you want to create " + nameWithoutExtension + " job?", "Are you sure you want to create " + nameWithoutExtension + " job?", "Are you sure you do not want to create " + nameWithoutExtension + " job?", ( ) => { string query = File.ReadAllText( queryFilename ); _ConsoleBuffer.Add( string.Format( "Creating {0} Stream Analytics job...", nameWithoutExtension ) ); CreateStreamAnalyticsJob( nameWithoutExtension, query, resourceGroupName, azurePrepIn, azurePrepOut.ehDevices, ehOutput ); }, _ConsoleBuffer ); } } }
private string SelectResourceGroup( AzurePrepInputs inputs ) { Console.WriteLine( "Retrieving a list of Resource Groups..." ); ResourceGroupExtended[] groups = AzureProvider.GetResourceGroups( inputs.Credentials ); int count = groups.Length; Console.WriteLine( "Available Resource Groups: " ); Console.WriteLine( "0: Create new Resource Group." ); for( int current = 1; current <= count; ++current ) { Console.WriteLine( current + ": " + groups[ current - 1 ].Name ); } for( ;; ) { Console.WriteLine( "Please select Resource Group from list: " ); string answer = Console.ReadLine( ); int selection = 0; if( !int.TryParse( answer, out selection ) || selection > count || selection < 0 ) { Console.WriteLine( "Incorrect Resource Group number." ); continue; } if( selection == 0 ) { if( ConsoleHelper.Confirm( "Are you sure you want to create new Resource Group?" ) ) { string resourceGroupName; for( ;; ) { Console.WriteLine( "Enter a name for Resource Group (only letters and digits, less than 17 chars long)." ); Console.WriteLine( "(Note that fully qualified path may also be subject to further length restrictions.)" ); resourceGroupName = Console.ReadLine( ); if( string.IsNullOrEmpty( resourceGroupName ) || !CheckNamePrefix( resourceGroupName ) ) { Console.WriteLine( "Namespace prefix should contain only letters and digits and have length less than 17." ); continue; } if( ConsoleHelper.Confirm( "Are you sure you want to create a Resource Group called " + resourceGroupName + "?" ) ) { break; } } AzureProvider.CreateResourceGroup( inputs.Credentials, resourceGroupName, inputs.Location ); return resourceGroupName; } } else { if( ConsoleHelper.Confirm( "Are you sure you want to select Resource Group " + groups[ selection - 1 ].Name + "?" ) ) { return groups[ selection - 1 ].Name; } } } }
private AzurePrepOutputs CreateEventHub( AzurePrepInputs inputs ) { AzurePrepOutputs result = new AzurePrepOutputs { SBNamespace = inputs.SBNamespace }; // Create Namespace var sbMgmt = new ServiceBusManagementClient( inputs.Credentials ); ServiceBusNamespaceResponse nsResponse = null; _ConsoleBuffer.Add( string.Format( "Creating Service Bus namespace {0} in location {1}", inputs.SBNamespace, inputs.Location ) ); try { // There is (currently) no clean error code returned when the namespace already exists // Check if it does nsResponse = sbMgmt.Namespaces.Create( inputs.SBNamespace, inputs.Location ); _ConsoleBuffer.Add( string.Format( "Service Bus namespace {0} created.", inputs.SBNamespace ) ); } catch ( Exception ) { nsResponse = null; _ConsoleBuffer.Add( string.Format( "Service Bus namespace {0} already existed.", inputs.SBNamespace ) ); } int triesCount = 0; // Wait until the namespace is active while( nsResponse == null || nsResponse.Namespace.Status != "Active" ) { nsResponse = sbMgmt.Namespaces.Get( inputs.SBNamespace ); if( nsResponse.Namespace.Status == "Active" ) { break; } triesCount += 1; if( triesCount % 10 == 0 ) { _ConsoleBuffer.Add( "Please note that activation could last about an hour if namespace with the same name prefix was deleted recently..." ); } else { _ConsoleBuffer.Add( string.Format( "Namespace {0} in state {1}. Waiting...", inputs.SBNamespace, nsResponse.Namespace.Status ) ); } System.Threading.Thread.Sleep( 5000 ); } // Get the namespace connection string var nsDescription = sbMgmt.Namespaces.GetNamespaceDescription( inputs.SBNamespace ); result.nsConnectionString = nsDescription.NamespaceDescriptions.First( ( d ) => String.Equals( d.AuthorizationType, "SharedAccessAuthorization" ) ).ConnectionString; // Create EHs + device keys + consumer keys (WebSite*) var nsManager = NamespaceManager.CreateFromConnectionString( result.nsConnectionString ); var ehDescriptionDevices = new EventHubDescription( inputs.EventHubNameDevices ) { PartitionCount = 8, }; ehDescriptionDevices.Authorization.Add( new SharedAccessAuthorizationRule( "D1", new List<AccessRights> { AccessRights.Send } ) ); ehDescriptionDevices.Authorization.Add( new SharedAccessAuthorizationRule( "D2", new List<AccessRights> { AccessRights.Send } ) ); ehDescriptionDevices.Authorization.Add( new SharedAccessAuthorizationRule( "D3", new List<AccessRights> { AccessRights.Send } ) ); ehDescriptionDevices.Authorization.Add( new SharedAccessAuthorizationRule( "D4", new List<AccessRights> { AccessRights.Send } ) ); ehDescriptionDevices.Authorization.Add( new SharedAccessAuthorizationRule( "WebSite", new List<AccessRights> { AccessRights.Manage, AccessRights.Listen, AccessRights.Send } ) ); ehDescriptionDevices.Authorization.Add( new SharedAccessAuthorizationRule( "StreamingAnalytics", new List<AccessRights> { AccessRights.Manage, AccessRights.Listen, AccessRights.Send } ) ); _ConsoleBuffer.Add( string.Format( "Creating Event Hub {0}...", inputs.EventHubNameDevices ) ); result.ehDevices = null; do { try { result.ehDevices = nsManager.CreateEventHubIfNotExists( ehDescriptionDevices ); } catch ( UnauthorizedAccessException ) { _ConsoleBuffer.Add( "Service Bus connection string not valid yet. Waiting..." ); System.Threading.Thread.Sleep( 5000 ); } } while ( result.ehDevices == null ); ConsoleHelper.AskAndPerformAction( "Do you want to create " + inputs.EventHubNameAlerts + " Event Hub?", "Are you sure you want to create " + inputs.EventHubNameAlerts + " Event Hub?", "Are you sure you do not want to create " + inputs.EventHubNameAlerts + " Event Hub?", ( ) => { var ehDescriptionAlerts = new EventHubDescription( inputs.EventHubNameAlerts ) { PartitionCount = 8, }; ehDescriptionAlerts.Authorization.Add( new SharedAccessAuthorizationRule( "WebSite", new List<AccessRights> { AccessRights.Manage, AccessRights.Listen, AccessRights.Send } ) ); ehDescriptionAlerts.Authorization.Add( new SharedAccessAuthorizationRule( "StreamingAnalytics", new List<AccessRights> { AccessRights.Manage, AccessRights.Listen, AccessRights.Send } ) ); _ConsoleBuffer.Add( string.Format( "Creating Event Hub {0}...", inputs.EventHubNameAlerts ) ); result.ehAlerts = null; do { try { result.ehAlerts = nsManager.CreateEventHubIfNotExists( ehDescriptionAlerts ); } catch ( UnauthorizedAccessException ) { _ConsoleBuffer.Add( "Service Bus connection string not valid yet. Waiting..." ); System.Threading.Thread.Sleep( 5000 ); } } while ( result.ehAlerts == null ); }, _ConsoleBuffer ); // Create Storage Account for Event Hub Processor var stgMgmt = new StorageManagementClient( inputs.Credentials ); try { _ConsoleBuffer.Add( string.Format( "Creating Storage Account {0} in location {1}...", inputs.StorageAccountName, inputs.Location ) ); var resultStg = stgMgmt.StorageAccounts.Create( new StorageAccountCreateParameters { Name = inputs.StorageAccountName.ToLowerInvariant(), Location = inputs.Location, AccountType = "Standard_LRS" } ); if( resultStg.StatusCode != System.Net.HttpStatusCode.OK ) { _ConsoleBuffer.Add( string.Format( "Error creating storage account {0} in Location {1}: {2}", inputs.StorageAccountName, inputs.Location, resultStg.StatusCode ) ); return null; } } catch( CloudException ce ) { if( String.Equals( ce.Error.Code, "ConflictError", StringComparison.InvariantCultureIgnoreCase ) ) { _ConsoleBuffer.Add( string.Format( "Storage account {0} already existed.", inputs.StorageAccountName ) ); } else { throw; } } return result; }
private string SelectRegion( AzurePrepInputs inputs ) { Console.WriteLine( "Retrieving a list of Locations..." ); string[] regions = AzureProvider.GetRegions( inputs.Credentials ); int regionsCount = regions.Length; Console.WriteLine( "Available locations: " ); for( int currentRegion = 1; currentRegion <= regionsCount; ++currentRegion ) { string suffixMessage = string.Empty; if( regions[currentRegion - 1] == "East US" ) { //see https://github.com/MSOpenTech/connectthedots/issues/168 suffixMessage = " (creating new Resource Group is not supported)"; } Console.WriteLine( currentRegion + ": " + regions[ currentRegion - 1 ] + suffixMessage ); } for( ;; ) { Console.WriteLine( "Please select Location from list: " ); string answer = Console.ReadLine( ); int selection = 0; if( !int.TryParse( answer, out selection ) || selection > regionsCount || selection < 1 ) { Console.WriteLine( "Incorrect Location number." ); continue; } if( ConsoleHelper.Confirm( "Are you sure you want to select location " + regions[selection - 1] + "?" ) ) { return regions[ selection - 1 ]; } } }