public async Task GetDatasetAsync() { Mock <AutoMl.AutoMlClient> mockGrpcClient = new Mock <AutoMl.AutoMlClient>(MockBehavior.Strict); mockGrpcClient.Setup(x => x.CreateOperationsClient()) .Returns(new Mock <Operations.OperationsClient>().Object); GetDatasetRequest expectedRequest = new GetDatasetRequest { DatasetName = new DatasetName("[PROJECT]", "[LOCATION]", "[DATASET]"), }; Dataset expectedResponse = new Dataset { Name = "name2-1052831874", DisplayName = "displayName1615086568", Description = "description-1724546052", ExampleCount = 1517063674, Etag = "etag3123477", }; mockGrpcClient.Setup(x => x.GetDatasetAsync(expectedRequest, It.IsAny <CallOptions>())) .Returns(new Grpc.Core.AsyncUnaryCall <Dataset>(Task.FromResult(expectedResponse), null, null, null, null)); AutoMlClient client = new AutoMlClientImpl(mockGrpcClient.Object, null); DatasetName name = new DatasetName("[PROJECT]", "[LOCATION]", "[DATASET]"); Dataset response = await client.GetDatasetAsync(name); Assert.Same(expectedResponse, response); mockGrpcClient.VerifyAll(); }
public BPM( string[] labels, double sparsityApproxThresh) { Debug.Assert(labels != null, "The labels must not be null."); Debug.Assert(labels.Length == 2, "The labels must have two possible values."); Debug.Assert(sparsityApproxThresh >= 0, "The sparsityApproxThresh must be greater than or equal to zero."); // Initialise the validations _validate = new Validate(); // Initialise the BPM _engine = new Dictionary <DistributionType, InferenceEngine>(); _w = new Dictionary <DistributionType, Variable <Vector> >(); _w[DistributionType.Prior] = null; _w[DistributionType.Posterior] = null; _d = new Dictionary <DistributionType, DistributionName>(); _yPredicDistrib = Enumerable.Empty <IDictionary <string, double> >(); _yPredicLabel = new string[] { }; _mapping = new GenericClassifierMapping(labels); // TO DO // Evaluator mapping var evaluatorMapping = _mapping.ForEvaluation(); _evaluator = new ClassifierEvaluator <IList <Vector>, int, IList <string>, string>(evaluatorMapping); // Other initialisations _availableDatasetName = new DatasetName(); _numObservations = 0; _numFeatures = 0; }
public void Dataset( string[] labels, DatasetName datasetName, object[,] x, object[] y) { if (x == null || y == null || x.Rank != 2 || x.GetLength(0) == 0 || y.Length == 0) { TraceListeners.Log(TraceEventType.Error, 0, "Invalid number of records in the " + datasetName.ToString() + " dataset!", true, true); } if (x.GetLength(1) != y.Count()) { TraceListeners.Log(TraceEventType.Error, 0, "Number of observations in features and target do not match!", true, true); } if (y.All(v => (string)v != labels[0]) || y.All(v => (string)v != labels[1])) { TraceListeners.Log(TraceEventType.Error, 0, "All the labels must be present in the " + datasetName.ToString() + " dataset!", true, true); } if (x.GetLength(1) < x.GetLength(0)) { TraceListeners.Log(TraceEventType.Error, 0, "Number of observations is less than number of features!", true, true); } }
public void GetDatasetResourceNames() { moq::Mock <DatasetService.DatasetServiceClient> mockGrpcClient = new moq::Mock <DatasetService.DatasetServiceClient>(moq::MockBehavior.Strict); mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object); GetDatasetRequest request = new GetDatasetRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), }; Dataset expectedResponse = new Dataset { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), DisplayName = "display_name137f65c2", MetadataSchemaUri = "metadata_schema_uric874bf0a", CreateTime = new wkt::Timestamp(), UpdateTime = new wkt::Timestamp(), Etag = "etage8ad7218", Labels = { { "key8a0b6e3c", "value60c16320" }, }, Metadata = new wkt::Value(), EncryptionSpec = new EncryptionSpec(), }; mockGrpcClient.Setup(x => x.GetDataset(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(expectedResponse); DatasetServiceClient client = new DatasetServiceClientImpl(mockGrpcClient.Object, null); Dataset response = client.GetDataset(request.DatasetName); xunit::Assert.Same(expectedResponse, response); mockGrpcClient.VerifyAll(); }
/// <summary>Snippet for DeleteDataset</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public void DeleteDatasetRequestObject() { // Create client AutoMlClient autoMlClient = AutoMlClient.Create(); // Initialize request argument(s) DeleteDatasetRequest request = new DeleteDatasetRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), }; // Make the request Operation <Empty, OperationMetadata> response = autoMlClient.DeleteDataset(request); // Poll until the returned long-running operation is complete Operation <Empty, OperationMetadata> completedResponse = response.PollUntilCompleted(); // Retrieve the operation result Empty result = completedResponse.Result; // Or get the name of the operation string operationName = response.Name; // This name can be stored, then the long-running operation retrieved later by name Operation <Empty, OperationMetadata> retrievedResponse = autoMlClient.PollOnceDeleteDataset(operationName); // Check if the retrieved long-running operation has completed if (retrievedResponse.IsCompleted) { // If it has completed, then access the result Empty retrievedResult = retrievedResponse.Result; } }
/// <summary>Snippet for ImportDataAsync</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public async Task ImportDataRequestObjectAsync() { // Create client AutoMlClient autoMlClient = await AutoMlClient.CreateAsync(); // Initialize request argument(s) ImportDataRequest request = new ImportDataRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), InputConfig = new InputConfig(), }; // Make the request Operation <Empty, OperationMetadata> response = await autoMlClient.ImportDataAsync(request); // Poll until the returned long-running operation is complete Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync(); // Retrieve the operation result Empty result = completedResponse.Result; // Or get the name of the operation string operationName = response.Name; // This name can be stored, then the long-running operation retrieved later by name Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceImportDataAsync(operationName); // Check if the retrieved long-running operation has completed if (retrievedResponse.IsCompleted) { // If it has completed, then access the result Empty retrievedResult = retrievedResponse.Result; } }
/// <summary>Snippet for GetDataset</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public void GetDatasetResourceNames() { // Create client AutoMlClient autoMlClient = AutoMlClient.Create(); // Initialize request argument(s) DatasetName name = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"); // Make the request Dataset response = autoMlClient.GetDataset(name); }
public virtual void InitialiseFeatures( DatasetName datasetName, object[,] features, object[] target) { TraceListeners.Log(TraceEventType.Information, 0, "InitialiseVariables...", false, true); _model.InitialiseFeatures(datasetName, features, target); }
/// <summary> /// Do the lookup! /// </summary> protected override void ProcessRecord() { // Setup for verbosity if we need it. var listener = new PSListener(this); Trace.Listeners.Add(listener); try { PandaTask t = null; bool needdatasets = OutputContainerNames.IsPresent || InputContainerNames.IsPresent; if (ParameterSetName == "TaskID") { t = TaskID.FindPandaJobWithTaskName(needdatasets); } if (ParameterSetName == "TaskObject") { t = PandaTaskObject.ID.FindPandaJobWithTaskName(needdatasets); } if (ParameterSetName == "TaskName") { t = TaskName.FindPandaJobWithTaskName(needdatasets); } if (ParameterSetName == "DatasetGRIDJob") { // Get the job and resulting dataset name. var job = JobParser.FindJob(JobName, JobVersion); var ds = job.ResultingDataSetName(DatasetName.Trim(), JobIteration); // Now, look up the job itself. t = (ds + "/").FindPandaJobWithTaskName(); } // We really can't deal with a bad task below. The returned objects are sufficiently complex. if (t == null) { throw new ArgumentException("Unable to find the task in panda: was it ever submitted?"); } // Dump the info to the output pipeline as requested. if (JobStatus.IsPresent) { WriteObject(t.status); } if (InputContainerNames.IsPresent) { WriteObject(t.DataSetNamesIN()); } if (OutputContainerNames.IsPresent) { WriteObject(t.DataSetNamesOUT()); } } finally { Trace.Listeners.Remove(listener); } }
/// <summary> /// Get the dataset name without extension, /// used for running RunParallelActiveLearning in the ActiveLearning Class /// </summary> /// <returns></returns> public string GetDataSetNameWithoutExtension() { string dataSetNameWithoutExtension = ""; int fileExtPos = DatasetName.LastIndexOf("."); if (fileExtPos >= 0) { dataSetNameWithoutExtension = DatasetName.Substring(0, fileExtPos); } return(dataSetNameWithoutExtension); }
/// <summary>Snippet for GetDataset</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public void GetDatasetRequestObject() { // Create client AutoMlClient autoMlClient = AutoMlClient.Create(); // Initialize request argument(s) GetDatasetRequest request = new GetDatasetRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), }; // Make the request Dataset response = autoMlClient.GetDataset(request); }
public object[] GetTargetConverted(DatasetName datasetName) { // Evaluate if (!_targetConverted.ContainsKey(datasetName)) { TraceListeners.Log(TraceEventType.Error, 0, "Dataset does not exist: " + datasetName.ToString(), true, true); } // Get return(_targetConverted[datasetName]); }
public string[,] GetDataset(DatasetName datasetName) { // Evaluate if (!_datasets.ContainsKey(datasetName)) { TraceListeners.Log(TraceEventType.Error, 0, "Dataset does not exist: " + datasetName.ToString(), true, true); } // Get return(_datasets[datasetName]); }
private bool AvailableDataset( DatasetName availableDatasetName, DatasetName datasetName) { if (availableDatasetName != datasetName) { TraceListeners.Log(TraceEventType.Error, 0, "The requested database is not valid or is not loaed into the model!", true, true); return(false); } return(true); }
/// <summary> /// Fetch a particular dataset. /// </summary> protected override void ProcessRecord() { var listener = new PSListener(this); Trace.Listeners.Add(listener); try { // Get the dataset name that we are to process. Depending on parameters, it can come // from one of two places. var dataset = ParameterSetName == "job" ? GetJobDatasetName() : DatasetName.Trim(); // Find all the members of this dataset. var allFilesToCopy = DataSetManager.ListOfFilesInDataSetAsync(dataset, m => DisplayStatus($"Listing Files in {dataset}", m), failNow: () => Stopping).Result; if (nFiles != 0) { allFilesToCopy = allFilesToCopy .OrderBy(u => u.AbsolutePath) .Take(nFiles) .ToArray(); } // If we have a location, we want to do a copy. If we don't have a location, then we just // want to get the files somewhere local. var loc = Location.AsIPlace().Result; //WriteError(new ErrorRecord(err, "NoSuchLocation", ErrorCategory.InvalidArgument, null)); var resultUris = loc == null ? DataSetManager.MakeFilesLocalAsync(allFilesToCopy, m => DisplayStatus($"Downloading {dataset}", m), failNow : () => Stopping, timeout : Timeout).Result : DataSetManager.CopyFilesToAsync(loc, allFilesToCopy, m => DisplayStatus($"Downloading {dataset}", m), failNow: () => Stopping, timeout: Timeout).Result; // Dump all the returned files out to whatever is next in the pipeline. if (PassThru.IsPresent) { using (var pl = listener.PauseListening()) { foreach (var ds in resultUris) { WriteObject(ds); } } } } finally { ClearStatus(); Trace.Listeners.Remove(listener); } }
public override void InitialiseFeatures( DatasetName datasetName, object[,] x, object[] y) { Debug.Assert(x != null, "The feature vector must not be null."); Debug.Assert(y != null, "The targe variable must not be null."); Vector[] xV = null; Range r = null; // Validate _validate.Dataset( labels: _mapping.GetClassLabels().ToArray(), datasetName: datasetName, x: x, y: y); // Set meta data _numFeatures = x.GetLength(0); _numObservations = x.GetLength(1); // Transpose double[][] xTrans = new double[_numObservations][]; for (int j = 0; j < _numObservations; j++) { xTrans[j] = new double[_numFeatures]; for (int i = 0; i < _numFeatures; i++) { xTrans[j][i] = (double)x[i, j]; } } // Set target _y = Variable.Observed( Array.ConvertAll(y, v => ( System.Convert.ToInt64(v) > 0) ? true : false)).Named( "y." + datasetName.ToString()); // Set features xV = new Vector[_numObservations]; r = _y.Range.Named("person"); for (int i = 0; i < _numObservations; i++) { xV[i] = Vector.FromArray(xTrans[i]); } _x = Variable.Observed(xV, r).Named("x." + datasetName.ToString()); _availableDatasetName = datasetName; }
public void RemoveFeature(DatasetName datasetName) { if (_datasets.ContainsKey(datasetName)) { _datasets.Remove(datasetName); // Reclaim the memory reserved if (_datasets.Count == 0) { _datasets.Clear(); _datasets = null; _datasets = new Dictionary <DatasetName, string[, ]>(); } } }
/// <summary>Snippet for ListTableSpecsAsync</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public async Task ListTableSpecsRequestObjectAsync() { // Create client AutoMlClient autoMlClient = await AutoMlClient.CreateAsync(); // Initialize request argument(s) ListTableSpecsRequest request = new ListTableSpecsRequest { ParentAsDatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), FieldMask = new FieldMask(), Filter = "", }; // Make the request PagedAsyncEnumerable <ListTableSpecsResponse, TableSpec> response = autoMlClient.ListTableSpecsAsync(request); // Iterate over all response items, lazily performing RPCs as required await response.ForEachAsync((TableSpec item) => { // Do something with each item Console.WriteLine(item); }); // Or iterate over pages (of server-defined size), performing one RPC per page await response.AsRawResponses().ForEachAsync((ListTableSpecsResponse page) => { // Do something with each page of items Console.WriteLine("A page of results:"); foreach (TableSpec item in page) { // Do something with each item Console.WriteLine(item); } }); // Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required int pageSize = 10; Page <TableSpec> singlePage = await response.ReadPageAsync(pageSize); // Do something with the page of items Console.WriteLine($"A page of {pageSize} results (unless it's the final page):"); foreach (TableSpec item in singlePage) { // Do something with each item Console.WriteLine(item); } // Store the pageToken, for when the next page is required. string nextPageToken = singlePage.NextPageToken; }
/// <summary> /// Get train and test datsets. /// </summary> public void ReadInputData( string modelName, string modelProcedure, string modellingGroup, string submodel, string submodelCondValue, DatasetName datasetName, Dictionary <DatasetName, string> samplingsCombDatabase, Dictionary <DatasetName, string> samplingsCombSample) { TraceListeners.Log(TraceEventType.Information, 0, "ClassifierMain::ReadInputData...", false, true); string query; string[] featureNames; // Read query = "CALL " + Constants.MYSQL_DATABASES_MYSQL[samplingsCombDatabase[datasetName]] + "." + modelName + "(\"" + modelProcedure + "\", \"" + modellingGroup + "\", \"" + submodel + "\"," + submodelCondValue + ",\"" + samplingsCombSample[datasetName] + "\");"; ReadMySQL reader = new ReadMySQL( samplingsCombDatabase[datasetName], Constants.MYSQL_SERVER, Constants.MYSQL_USERNAME, Constants.MYSQL_PASSWORD, Constants.MYSQL_DATABASES_MYSQL[samplingsCombDatabase[datasetName]], Constants.MYSQL_CMD_TIMEOUT, Constants.MYSQL_CONN_TIMEOUT, Constants.MYSQL_NULL_DEFAULT.ToString()); reader.Read(query); // Set featureNames = reader.GetColumnsNames(); _features.Set( dataset: reader.GetColumns(featureNames), datasetName: datasetName, featuresByOrder: featureNames); // Convert & Transform _features.ConvertFeatures( datasetName: datasetName, standardConversion: true); // Close reader.CloseConnection(); }
private object Convert( DatasetName datasetName, string featureName, int featureIndex, int observationIndex, string featureType = null) { object output = null; if (featureType == null) { featureType = _featuresTypes[featureName]; } switch (featureType.ToLower()) { case "bool": output = (System.Convert.ToInt64( _datasets[datasetName][featureIndex, observationIndex]) > 0) ? true : false; break; case "double": output = System.Convert.ToDouble( _datasets[datasetName][featureIndex, observationIndex]); break; case "int": output = System.Convert.ToInt32( _datasets[datasetName][featureIndex, observationIndex]); break; case "date": // break; case "string": output = _datasets[datasetName][featureIndex, observationIndex]; break; default: TraceListeners.Log(TraceEventType.Error, 0, "Invalid variable type for " + featureName + " in the metadata specification", true, true); break; } return(output); }
public override int GetHashCode() { int hash = 1; if (Id.Length != 0) { hash ^= Id.GetHashCode(); } if (Consumer.Length != 0) { hash ^= Consumer.GetHashCode(); } if (DatasetName.Length != 0) { hash ^= DatasetName.GetHashCode(); } if (DatasetType.Length != 0) { hash ^= DatasetType.GetHashCode(); } if (DatasetId.Length != 0) { hash ^= DatasetId.GetHashCode(); } hash ^= tags_.GetHashCode(); if (data_ != null) { hash ^= Data.GetHashCode(); } hash ^= targetValues_.GetHashCode(); if (model_ != null) { hash ^= Model.GetHashCode(); } hash ^= prediction_.GetHashCode(); if (TimeAdded != 0UL) { hash ^= TimeAdded.GetHashCode(); } if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } return(hash); }
public override void InitialiseFeatures( DatasetName datasetName, object[,] x, object[] y) { Debug.Assert(x != null, "The feature vector must not be null."); Debug.Assert(y != null, "The targe variable must not be null."); // Validate _validate.Dataset( labels: _mapping.GetClassLabels().ToArray(), datasetName: datasetName, x: x, y: y); // Set meta data _numFeatures = x.GetLength(0); _numObservations = x.GetLength(1); // Transpose double[][] xTrans = new double[_numObservations][]; for (int j = 0; j < _numObservations; j++) { xTrans[j] = new double[_numFeatures]; for (int i = 0; i < _numFeatures; i++) { xTrans[j][i] = (double)x[i, j]; } } // Set target _y = new List <string>( Array.ConvertAll(y, v => v.ToString())); // Set features _x = new Vector[_numObservations]; for (int i = 0; i < _numObservations; i++) { _x[i] = Vector.FromArray(xTrans[i]); } _availableDatasetName = datasetName; }
public override int GetHashCode() { int hash = 1; if (DatasetName.Length != 0) { hash ^= DatasetName.GetHashCode(); } hash ^= inputs_.GetHashCode(); if (options_ != null) { hash ^= Options.GetHashCode(); } if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } return(hash); }
/// <summary>Snippet for ListTableSpecs</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public void ListTableSpecsResourceNames() { // Create client AutoMlClient autoMlClient = AutoMlClient.Create(); // Initialize request argument(s) DatasetName parent = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"); // Make the request PagedEnumerable <ListTableSpecsResponse, TableSpec> response = autoMlClient.ListTableSpecs(parent); // Iterate over all response items, lazily performing RPCs as required foreach (TableSpec item in response) { // Do something with each item Console.WriteLine(item); } // Or iterate over pages (of server-defined size), performing one RPC per page foreach (ListTableSpecsResponse page in response.AsRawResponses()) { // Do something with each page of items Console.WriteLine("A page of results:"); foreach (TableSpec item in page) { // Do something with each item Console.WriteLine(item); } } // Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required int pageSize = 10; Page <TableSpec> singlePage = response.ReadPage(pageSize); // Do something with the page of items Console.WriteLine($"A page of {pageSize} results (unless it's the final page):"); foreach (TableSpec item in singlePage) { // Do something with each item Console.WriteLine(item); } // Store the pageToken, for when the next page is required. string nextPageToken = singlePage.NextPageToken; }
public void RemoveFeatureConverted(DatasetName datasetName) { if (_featuresConverted.ContainsKey(datasetName) & _targetConverted.ContainsKey(datasetName)) { _featuresConverted.Remove(datasetName); _targetConverted.Remove(datasetName); // Reclaim the memory reserved if (_featuresConverted.Count == 0) { _featuresConverted.Clear(); _featuresConverted = null; _featuresConverted = new Dictionary <DatasetName, object[, ]>(); _targetConverted.Clear(); _targetConverted = null; _targetConverted = new Dictionary <DatasetName, object[]>(); } } }
public async stt::Task GetDatasetRequestObjectAsync() { moq::Mock <DatasetService.DatasetServiceClient> mockGrpcClient = new moq::Mock <DatasetService.DatasetServiceClient>(moq::MockBehavior.Strict); mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object); GetDatasetRequest request = new GetDatasetRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), ReadMask = new wkt::FieldMask(), }; Dataset expectedResponse = new Dataset { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), DisplayName = "display_name137f65c2", MetadataSchemaUri = "metadata_schema_uric874bf0a", CreateTime = new wkt::Timestamp(), UpdateTime = new wkt::Timestamp(), Etag = "etage8ad7218", Labels = { { "key8a0b6e3c", "value60c16320" }, }, Metadata = new wkt::Value(), EncryptionSpec = new EncryptionSpec(), Description = "description2cf9da67", }; mockGrpcClient.Setup(x => x.GetDatasetAsync(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(new grpccore::AsyncUnaryCall <Dataset>(stt::Task.FromResult(expectedResponse), null, null, null, null)); DatasetServiceClient client = new DatasetServiceClientImpl(mockGrpcClient.Object, null); Dataset responseCallSettings = await client.GetDatasetAsync(request, gaxgrpc::CallSettings.FromCancellationToken(st::CancellationToken.None)); xunit::Assert.Same(expectedResponse, responseCallSettings); Dataset responseCancellationToken = await client.GetDatasetAsync(request, st::CancellationToken.None); xunit::Assert.Same(expectedResponse, responseCancellationToken); mockGrpcClient.VerifyAll(); }
public void Set( Dictionary <string, List <string> > dataset, DatasetName datasetName, string[] featuresByOrder) { TraceListeners.Log(TraceEventType.Information, 0, "Features::Set...", false, true); int numFeatures = dataset.Keys.Count; int numObservations = dataset[dataset.Keys.First()].Count; // Evaluate if (_datasets.ContainsKey(datasetName)) { TraceListeners.Log(TraceEventType.Warning, 0, "Overwriting: " + datasetName.ToString(), false, true); } // Set try { _featuresKey[datasetName] = new Dictionary <string, int>(); _datasets[datasetName] = new string[numFeatures, numObservations]; int i = 0; foreach (string k in featuresByOrder) { _featuresKey[datasetName][k] = i; for (int j = 0; j < numObservations; j++) { _datasets[datasetName][i, j] = dataset[k][j]; } i++; } } catch (Exception e) { TraceListeners.Log(TraceEventType.Error, 0, e.ToString(), true, true); } }
public BPMMapped( string[] labels) { Debug.Assert(labels != null, "The labels must not be null."); Debug.Assert(labels.Length == 2, "The labels must have two possible values."); // Initialise the validations _validate = new Validate(); // Create a BPM from the mapping _mapping = new GenericClassifierMapping(labels); _classifier = BayesPointMachineClassifier.CreateBinaryClassifier(_mapping); // Evaluator mapping var evaluatorMapping = _mapping.ForEvaluation(); _evaluator = new ClassifierEvaluator <IList <Vector>, int, IList <string>, string>(evaluatorMapping); // Other initialisations _availableDatasetName = new DatasetName(); _numObservations = 0; _numFeatures = 0; }
/// <summary> /// Returns a hash code for this instance. /// </summary> /// <returns> /// A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table. /// </returns> public override int GetHashCode() { return(DatasetName.GetHashCode()); }
/// <summary> /// Fetch a particular dataset. /// </summary> protected override void ProcessRecord() { var listener = new PSListener(this); Trace.Listeners.Add(listener); try { // Get the actual dataset name. var dataset = DatasetName.Trim(); if (ParameterSetName == "job") { // Get the job, see if it is finished, and then get the output dataset. var job = JobParser.FindJob(JobName, JobVersion); if (job == null) { throw new ArgumentException($"Job {JobName} v{JobVersion} is not known to the system!"); } // Get the resulting job name for this guy. var pandaJobName = job.ResultingDataSetName(JobSourceDatasetName, JobIteration) + "/"; // Now, to get the output dataset, we need to fetch the job. var pandaTask = pandaJobName.FindPandaJobWithTaskName(true); if (pandaTask == null) { throw new ArgumentException($"No panda task found with name '{pandaJobName}' for job '{JobName}' v{JobVersion}."); } var containers = pandaTask.DataSetNamesOUT(); if (containers.Length > 1) { throw new ArgumentException($"There are more than one output container for the panda task {pandaTask.jeditaskid} - can't decide. Need code upgrade!! Thanks for the fish!"); } dataset = containers.First(); } // Find all the members of this dataset. var allFilesToCopy = DataSetManager.ListOfFilesInDataSetAsync(dataset, m => DisplayStatus($"Listing Files in {dataset}", m), failNow: () => Stopping) .Result; if (nFiles != 0) { allFilesToCopy = allFilesToCopy .OrderBy(u => u.AbsolutePath) .Take(nFiles) .ToArray(); } // Turn the source and destination locations into actual locations. var locSource = SourceLocation.AsIPlace().Result; var locDest = DestinationLocation.AsIPlace().Result; // Do the actual copy. This will fail if one of the files can't be found at the source. var resultUris = DataSetManager .CopyFilesAsync(locSource, locDest, allFilesToCopy, mbox => DisplayStatus($"Downloading {dataset}", mbox), failNow: () => Stopping, timeout: Timeout) .Result; // Dump all the returned files out to whatever is next in the pipeline. if (PassThru.IsPresent) { using (var pl = listener.PauseListening()) { foreach (var ds in resultUris) { WriteObject(ds); } } } } finally { Trace.Listeners.Remove(listener); } }