/// <summary> /// Unmarshaller the response from the service to the response class. /// </summary> /// <param name="context"></param> /// <returns></returns> public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { DescribeTrainingJobResponse response = new DescribeTrainingJobResponse(); context.Read(); int targetDepth = context.CurrentDepth; while (context.ReadAtDepth(targetDepth)) { if (context.TestExpression("AlgorithmSpecification", targetDepth)) { var unmarshaller = AlgorithmSpecificationUnmarshaller.Instance; response.AlgorithmSpecification = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("CreationTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.CreationTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("FailureReason", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.FailureReason = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("HyperParameters", targetDepth)) { var unmarshaller = new DictionaryUnmarshaller <string, string, StringUnmarshaller, StringUnmarshaller>(StringUnmarshaller.Instance, StringUnmarshaller.Instance); response.HyperParameters = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("InputDataConfig", targetDepth)) { var unmarshaller = new ListUnmarshaller <Channel, ChannelUnmarshaller>(ChannelUnmarshaller.Instance); response.InputDataConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("LastModifiedTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.LastModifiedTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("ModelArtifacts", targetDepth)) { var unmarshaller = ModelArtifactsUnmarshaller.Instance; response.ModelArtifacts = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("OutputDataConfig", targetDepth)) { var unmarshaller = OutputDataConfigUnmarshaller.Instance; response.OutputDataConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("ResourceConfig", targetDepth)) { var unmarshaller = ResourceConfigUnmarshaller.Instance; response.ResourceConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("RoleArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.RoleArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("SecondaryStatus", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.SecondaryStatus = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("SecondaryStatusTransitions", targetDepth)) { var unmarshaller = new ListUnmarshaller <SecondaryStatusTransition, SecondaryStatusTransitionUnmarshaller>(SecondaryStatusTransitionUnmarshaller.Instance); response.SecondaryStatusTransitions = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("StoppingCondition", targetDepth)) { var unmarshaller = StoppingConditionUnmarshaller.Instance; response.StoppingCondition = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingEndTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.TrainingEndTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingJobArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TrainingJobArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingJobName", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TrainingJobName = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingJobStatus", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TrainingJobStatus = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingStartTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.TrainingStartTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TuningJobArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TuningJobArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("VpcConfig", targetDepth)) { var unmarshaller = VpcConfigUnmarshaller.Instance; response.VpcConfig = unmarshaller.Unmarshall(context); continue; } } return(response); }
/// <summary> /// Unmarshaller the response from the service to the response class. /// </summary> /// <param name="context"></param> /// <returns></returns> public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { DescribeTrainingJobResponse response = new DescribeTrainingJobResponse(); context.Read(); int targetDepth = context.CurrentDepth; while (context.ReadAtDepth(targetDepth)) { if (context.TestExpression("AlgorithmSpecification", targetDepth)) { var unmarshaller = AlgorithmSpecificationUnmarshaller.Instance; response.AlgorithmSpecification = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("AutoMLJobArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.AutoMLJobArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("BillableTimeInSeconds", targetDepth)) { var unmarshaller = IntUnmarshaller.Instance; response.BillableTimeInSeconds = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("CheckpointConfig", targetDepth)) { var unmarshaller = CheckpointConfigUnmarshaller.Instance; response.CheckpointConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("CreationTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.CreationTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("DebugHookConfig", targetDepth)) { var unmarshaller = DebugHookConfigUnmarshaller.Instance; response.DebugHookConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("DebugRuleConfigurations", targetDepth)) { var unmarshaller = new ListUnmarshaller <DebugRuleConfiguration, DebugRuleConfigurationUnmarshaller>(DebugRuleConfigurationUnmarshaller.Instance); response.DebugRuleConfigurations = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("DebugRuleEvaluationStatuses", targetDepth)) { var unmarshaller = new ListUnmarshaller <DebugRuleEvaluationStatus, DebugRuleEvaluationStatusUnmarshaller>(DebugRuleEvaluationStatusUnmarshaller.Instance); response.DebugRuleEvaluationStatuses = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("EnableInterContainerTrafficEncryption", targetDepth)) { var unmarshaller = BoolUnmarshaller.Instance; response.EnableInterContainerTrafficEncryption = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("EnableManagedSpotTraining", targetDepth)) { var unmarshaller = BoolUnmarshaller.Instance; response.EnableManagedSpotTraining = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("EnableNetworkIsolation", targetDepth)) { var unmarshaller = BoolUnmarshaller.Instance; response.EnableNetworkIsolation = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("ExperimentConfig", targetDepth)) { var unmarshaller = ExperimentConfigUnmarshaller.Instance; response.ExperimentConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("FailureReason", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.FailureReason = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("FinalMetricDataList", targetDepth)) { var unmarshaller = new ListUnmarshaller <MetricData, MetricDataUnmarshaller>(MetricDataUnmarshaller.Instance); response.FinalMetricDataList = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("HyperParameters", targetDepth)) { var unmarshaller = new DictionaryUnmarshaller <string, string, StringUnmarshaller, StringUnmarshaller>(StringUnmarshaller.Instance, StringUnmarshaller.Instance); response.HyperParameters = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("InputDataConfig", targetDepth)) { var unmarshaller = new ListUnmarshaller <Channel, ChannelUnmarshaller>(ChannelUnmarshaller.Instance); response.InputDataConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("LabelingJobArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.LabelingJobArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("LastModifiedTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.LastModifiedTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("ModelArtifacts", targetDepth)) { var unmarshaller = ModelArtifactsUnmarshaller.Instance; response.ModelArtifacts = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("OutputDataConfig", targetDepth)) { var unmarshaller = OutputDataConfigUnmarshaller.Instance; response.OutputDataConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("ResourceConfig", targetDepth)) { var unmarshaller = ResourceConfigUnmarshaller.Instance; response.ResourceConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("RoleArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.RoleArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("SecondaryStatus", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.SecondaryStatus = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("SecondaryStatusTransitions", targetDepth)) { var unmarshaller = new ListUnmarshaller <SecondaryStatusTransition, SecondaryStatusTransitionUnmarshaller>(SecondaryStatusTransitionUnmarshaller.Instance); response.SecondaryStatusTransitions = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("StoppingCondition", targetDepth)) { var unmarshaller = StoppingConditionUnmarshaller.Instance; response.StoppingCondition = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TensorBoardOutputConfig", targetDepth)) { var unmarshaller = TensorBoardOutputConfigUnmarshaller.Instance; response.TensorBoardOutputConfig = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingEndTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.TrainingEndTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingJobArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TrainingJobArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingJobName", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TrainingJobName = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingJobStatus", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TrainingJobStatus = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingStartTime", targetDepth)) { var unmarshaller = DateTimeUnmarshaller.Instance; response.TrainingStartTime = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TrainingTimeInSeconds", targetDepth)) { var unmarshaller = IntUnmarshaller.Instance; response.TrainingTimeInSeconds = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("TuningJobArn", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.TuningJobArn = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("VpcConfig", targetDepth)) { var unmarshaller = VpcConfigUnmarshaller.Instance; response.VpcConfig = unmarshaller.Unmarshall(context); continue; } } return(response); }
static void Main(string[] args) { string accesKeyId = ""; string accessKeySecret = ""; string trainingImage = ""; string roleArn = ""; string trainingJobName = $"ontology-training-job-{DateTime.UtcNow.Ticks}"; string endpointName = $"ontology-endpoint-{DateTime.UtcNow.Ticks}"; using (AmazonS3Client client = new AmazonS3Client(accesKeyId, accessKeySecret, Amazon.RegionEndpoint.EUCentral1)) { TransferUtility fileTransferUtility = new TransferUtility(client); // upload our csv training\test files using (AmazonSageMakerClient awsSageMakerClient = new AmazonSageMakerClient(accesKeyId, accessKeySecret, Amazon.RegionEndpoint.EUCentral1)) { CreateTrainingJobResponse response = awsSageMakerClient.CreateTrainingJobAsync(new CreateTrainingJobRequest() { AlgorithmSpecification = new AlgorithmSpecification() { TrainingInputMode = TrainingInputMode.File, TrainingImage = trainingImage }, OutputDataConfig = new OutputDataConfig() { S3OutputPath = "https://s3.eu-central-1.amazonaws.com/sagemaker-ovechko/sagemaker/test-csv/output" }, ResourceConfig = new ResourceConfig() { InstanceCount = 1, InstanceType = TrainingInstanceType.MlM4Xlarge, VolumeSizeInGB = 5 }, TrainingJobName = trainingJobName, HyperParameters = new Dictionary <string, string>() { { "eta", "0.1" }, { "objective", "multi:softmax" }, { "num_round", "5" }, { "num_class", "3" } }, StoppingCondition = new StoppingCondition() { MaxRuntimeInSeconds = 3600 }, RoleArn = roleArn, InputDataConfig = new List <Channel>() { new Channel() { ChannelName = "train", DataSource = new DataSource() { S3DataSource = new S3DataSource() { S3DataType = S3DataType.S3Prefix, S3Uri = "https://s3.eu-central-1.amazonaws.com/sagemaker-ovechko/sagemaker/test-csv/train/", S3DataDistributionType = S3DataDistribution.FullyReplicated } }, ContentType = "csv", CompressionType = Amazon.SageMaker.CompressionType.None }, new Channel() { ChannelName = "validation", DataSource = new DataSource() { S3DataSource = new S3DataSource() { S3DataType = S3DataType.S3Prefix, S3Uri = "https://s3.eu-central-1.amazonaws.com/sagemaker-ovechko/sagemaker/test-csv/validation/", S3DataDistributionType = S3DataDistribution.FullyReplicated } }, ContentType = "csv", CompressionType = Amazon.SageMaker.CompressionType.None } } }).Result; string modelName = $"{trainingJobName}-model"; DescribeTrainingJobResponse info = new DescribeTrainingJobResponse() { TrainingJobStatus = TrainingJobStatus.InProgress }; while (info.TrainingJobStatus == TrainingJobStatus.InProgress) { info = awsSageMakerClient.DescribeTrainingJobAsync(new DescribeTrainingJobRequest() { TrainingJobName = trainingJobName }).Result; if (info.TrainingJobStatus == TrainingJobStatus.InProgress) { Logger.Info("Training job creation is in progress..."); Thread.Sleep(10000); } } Logger.Info($"Training job creation has been finished. With status {info.TrainingJobStatus.ToString()}. {info.FailureReason}"); if (info.TrainingJobStatus == TrainingJobStatus.Completed) { CreateModelResponse modelCreationInfo = awsSageMakerClient.CreateModelAsync(new CreateModelRequest() { ModelName = modelName, ExecutionRoleArn = roleArn, PrimaryContainer = new ContainerDefinition() { ModelDataUrl = info.ModelArtifacts.S3ModelArtifacts, Image = trainingImage } }).Result; string endpointConfigName = $"{endpointName}-config"; awsSageMakerClient.CreateEndpointConfigAsync(new CreateEndpointConfigRequest() { EndpointConfigName = endpointConfigName, ProductionVariants = new List <ProductionVariant>() { new ProductionVariant() { InstanceType = ProductionVariantInstanceType.MlM4Xlarge, InitialVariantWeight = 1, InitialInstanceCount = 1, ModelName = modelName, VariantName = "AllTraffic" } } }); CreateEndpointResponse endpointCreationInfo = awsSageMakerClient.CreateEndpointAsync(new CreateEndpointRequest() { EndpointConfigName = endpointConfigName, EndpointName = endpointName }).Result; EndpointStatus currentStatus = EndpointStatus.Creating; while (currentStatus == EndpointStatus.Creating) { currentStatus = awsSageMakerClient.DescribeEndpointAsync(new DescribeEndpointRequest() { EndpointName = endpointName }).Result.EndpointStatus; if (currentStatus == EndpointStatus.Creating) { Logger.Info("Endpoint creation is in progress..."); Thread.Sleep(10000); } } Logger.Info("Endpoint creation has been finished."); if (currentStatus == EndpointStatus.InService) { using (AmazonSageMakerRuntimeClient sageMakerRuntimeClient = new AmazonSageMakerRuntimeClient(accesKeyId, accessKeySecret, Amazon.RegionEndpoint.EUCentral1)) { using (MemoryStream ms = new MemoryStream()) { GetObjectResponse s3Response = client.GetObjectAsync("sagemaker-ovechko", "sagemaker/test-csv/test/test.csv").Result; s3Response.ResponseStream.CopyTo(ms); ms.Seek(0, SeekOrigin.Begin); using (StreamReader sr = new StreamReader(ms)) { string csv = sr.ReadToEnd(); csv = csv.Replace("", string.Empty); using (MemoryStream ms2 = new MemoryStream(Encoding.ASCII.GetBytes(csv))) { InvokeEndpointResponse endpointResponseInfo = sageMakerRuntimeClient.InvokeEndpointAsync(new InvokeEndpointRequest() { ContentType = "text/csv", EndpointName = endpointName, Body = ms2, }).Result; using (StreamReader sr2 = new StreamReader(endpointResponseInfo.Body)) { string endpointResponseBody = sr2.ReadToEnd(); Logger.Info(endpointResponseBody); } } } } Logger.Info("Performing clean up..."); awsSageMakerClient.DeleteEndpointAsync(new DeleteEndpointRequest() { EndpointName = endpointName }); awsSageMakerClient.DeleteEndpointConfigAsync(new DeleteEndpointConfigRequest() { EndpointConfigName = endpointConfigName }); awsSageMakerClient.DeleteModelAsync(new DeleteModelRequest() { ModelName = modelName }); Logger.Info("Clean up finished."); } } } } } Console.ReadLine(); }