public async Task GetDatasetAsync()
{
Mock <AutoMl.AutoMlClient> mockGrpcClient = new Mock <AutoMl.AutoMlClient>(MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient())
.Returns(new Mock <Operations.OperationsClient>().Object);
GetDatasetRequest expectedRequest = new GetDatasetRequest
{
DatasetName = new DatasetName("[PROJECT]", "[LOCATION]", "[DATASET]"),
};
Dataset expectedResponse = new Dataset
{
Name = "name2-1052831874",
DisplayName = "displayName1615086568",
Description = "description-1724546052",
ExampleCount = 1517063674,
Etag = "etag3123477",
};
mockGrpcClient.Setup(x => x.GetDatasetAsync(expectedRequest, It.IsAny <CallOptions>()))
.Returns(new Grpc.Core.AsyncUnaryCall <Dataset>(Task.FromResult(expectedResponse), null, null, null, null));
AutoMlClient client = new AutoMlClientImpl(mockGrpcClient.Object, null);
DatasetName name = new DatasetName("[PROJECT]", "[LOCATION]", "[DATASET]");
Dataset response = await client.GetDatasetAsync(name);
Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
public BPM(
string[] labels,
double sparsityApproxThresh)
{
Debug.Assert(labels != null, "The labels must not be null.");
Debug.Assert(labels.Length == 2, "The labels must have two possible values.");
Debug.Assert(sparsityApproxThresh >= 0, "The sparsityApproxThresh must be greater than or equal to zero.");
// Initialise the validations
_validate = new Validate();
// Initialise the BPM
_engine = new Dictionary <DistributionType, InferenceEngine>();
_w = new Dictionary <DistributionType, Variable <Vector> >();
_w[DistributionType.Prior] = null;
_w[DistributionType.Posterior] = null;
_d = new Dictionary <DistributionType, DistributionName>();
_yPredicDistrib = Enumerable.Empty <IDictionary <string, double> >();
_yPredicLabel = new string[] { };
_mapping = new GenericClassifierMapping(labels);
// TO DO
// Evaluator mapping
var evaluatorMapping = _mapping.ForEvaluation();
_evaluator = new ClassifierEvaluator
<IList <Vector>, int, IList <string>, string>(evaluatorMapping);
// Other initialisations
_availableDatasetName = new DatasetName();
_numObservations = 0;
_numFeatures = 0;
}
public void Dataset(
string[] labels,
DatasetName datasetName,
object[,] x,
object[] y)
{
if (x == null ||
y == null ||
x.Rank != 2 ||
x.GetLength(0) == 0 ||
y.Length == 0)
{
TraceListeners.Log(TraceEventType.Error, 0,
"Invalid number of records in the " + datasetName.ToString() + " dataset!", true, true);
}
if (x.GetLength(1) != y.Count())
{
TraceListeners.Log(TraceEventType.Error, 0,
"Number of observations in features and target do not match!", true, true);
}
if (y.All(v => (string)v != labels[0]) ||
y.All(v => (string)v != labels[1]))
{
TraceListeners.Log(TraceEventType.Error, 0,
"All the labels must be present in the " + datasetName.ToString() + " dataset!", true, true);
}
if (x.GetLength(1) < x.GetLength(0))
{
TraceListeners.Log(TraceEventType.Error, 0,
"Number of observations is less than number of features!", true, true);
}
}
public void GetDatasetResourceNames()
{
moq::Mock <DatasetService.DatasetServiceClient> mockGrpcClient = new moq::Mock <DatasetService.DatasetServiceClient>(moq::MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
GetDatasetRequest request = new GetDatasetRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
};
Dataset expectedResponse = new Dataset
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
DisplayName = "display_name137f65c2",
MetadataSchemaUri = "metadata_schema_uric874bf0a",
CreateTime = new wkt::Timestamp(),
UpdateTime = new wkt::Timestamp(),
Etag = "etage8ad7218",
Labels =
{
{
"key8a0b6e3c",
"value60c16320"
},
},
Metadata = new wkt::Value(),
EncryptionSpec = new EncryptionSpec(),
};
mockGrpcClient.Setup(x => x.GetDataset(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(expectedResponse);
DatasetServiceClient client = new DatasetServiceClientImpl(mockGrpcClient.Object, null);
Dataset response = client.GetDataset(request.DatasetName);
xunit::Assert.Same(expectedResponse, response);
mockGrpcClient.VerifyAll();
}
/// <summary>Snippet for DeleteDataset</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void DeleteDatasetRequestObject()
{
// Create client
AutoMlClient autoMlClient = AutoMlClient.Create();
// Initialize request argument(s)
DeleteDatasetRequest request = new DeleteDatasetRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
};
// Make the request
Operation <Empty, OperationMetadata> response = autoMlClient.DeleteDataset(request);
// Poll until the returned long-running operation is complete
Operation <Empty, OperationMetadata> completedResponse = response.PollUntilCompleted();
// Retrieve the operation result
Empty result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Empty, OperationMetadata> retrievedResponse = autoMlClient.PollOnceDeleteDataset(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Empty retrievedResult = retrievedResponse.Result;
}
}
/// <summary>Snippet for ImportDataAsync</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public async Task ImportDataRequestObjectAsync()
{
// Create client
AutoMlClient autoMlClient = await AutoMlClient.CreateAsync();
// Initialize request argument(s)
ImportDataRequest request = new ImportDataRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
InputConfig = new InputConfig(),
};
// Make the request
Operation <Empty, OperationMetadata> response = await autoMlClient.ImportDataAsync(request);
// Poll until the returned long-running operation is complete
Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync();
// Retrieve the operation result
Empty result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceImportDataAsync(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Empty retrievedResult = retrievedResponse.Result;
}
}
/// <summary>Snippet for GetDataset</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void GetDatasetResourceNames()
{
// Create client
AutoMlClient autoMlClient = AutoMlClient.Create();
// Initialize request argument(s)
DatasetName name = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]");
// Make the request
Dataset response = autoMlClient.GetDataset(name);
}
public virtual void InitialiseFeatures(
DatasetName datasetName,
object[,] features,
object[] target)
{
TraceListeners.Log(TraceEventType.Information, 0,
"InitialiseVariables...", false, true);
_model.InitialiseFeatures(datasetName, features, target);
}
/// <summary>
/// Do the lookup!
/// </summary>
protected override void ProcessRecord()
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
PandaTask t = null;
bool needdatasets = OutputContainerNames.IsPresent || InputContainerNames.IsPresent;
if (ParameterSetName == "TaskID")
{
t = TaskID.FindPandaJobWithTaskName(needdatasets);
}
if (ParameterSetName == "TaskObject")
{
t = PandaTaskObject.ID.FindPandaJobWithTaskName(needdatasets);
}
if (ParameterSetName == "TaskName")
{
t = TaskName.FindPandaJobWithTaskName(needdatasets);
}
if (ParameterSetName == "DatasetGRIDJob")
{
// Get the job and resulting dataset name.
var job = JobParser.FindJob(JobName, JobVersion);
var ds = job.ResultingDataSetName(DatasetName.Trim(), JobIteration);
// Now, look up the job itself.
t = (ds + "/").FindPandaJobWithTaskName();
}
// We really can't deal with a bad task below. The returned objects are sufficiently complex.
if (t == null)
{
throw new ArgumentException("Unable to find the task in panda: was it ever submitted?");
}
// Dump the info to the output pipeline as requested.
if (JobStatus.IsPresent)
{
WriteObject(t.status);
}
if (InputContainerNames.IsPresent)
{
WriteObject(t.DataSetNamesIN());
}
if (OutputContainerNames.IsPresent)
{
WriteObject(t.DataSetNamesOUT());
}
} finally
{
Trace.Listeners.Remove(listener);
}
}
/// <summary>
/// Get the dataset name without extension,
/// used for running RunParallelActiveLearning in the ActiveLearning Class
/// </summary>
/// <returns></returns>
public string GetDataSetNameWithoutExtension()
{
string dataSetNameWithoutExtension = "";
int fileExtPos = DatasetName.LastIndexOf(".");
if (fileExtPos >= 0)
{
dataSetNameWithoutExtension = DatasetName.Substring(0, fileExtPos);
}
return(dataSetNameWithoutExtension);
}
/// <summary>Snippet for GetDataset</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void GetDatasetRequestObject()
{
// Create client
AutoMlClient autoMlClient = AutoMlClient.Create();
// Initialize request argument(s)
GetDatasetRequest request = new GetDatasetRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
};
// Make the request
Dataset response = autoMlClient.GetDataset(request);
}
public object[] GetTargetConverted(DatasetName datasetName)
{
// Evaluate
if (!_targetConverted.ContainsKey(datasetName))
{
TraceListeners.Log(TraceEventType.Error, 0,
"Dataset does not exist: " + datasetName.ToString(), true, true);
}
// Get
return(_targetConverted[datasetName]);
}
public string[,] GetDataset(DatasetName datasetName)
{
// Evaluate
if (!_datasets.ContainsKey(datasetName))
{
TraceListeners.Log(TraceEventType.Error, 0,
"Dataset does not exist: " + datasetName.ToString(), true, true);
}
// Get
return(_datasets[datasetName]);
}
private bool AvailableDataset(
DatasetName availableDatasetName,
DatasetName datasetName)
{
if (availableDatasetName != datasetName)
{
TraceListeners.Log(TraceEventType.Error, 0,
"The requested database is not valid or is not loaed into the model!",
true, true);
return(false);
}
return(true);
}
/// <summary>
/// Fetch a particular dataset.
/// </summary>
protected override void ProcessRecord()
{
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Get the dataset name that we are to process. Depending on parameters, it can come
// from one of two places.
var dataset = ParameterSetName == "job"
? GetJobDatasetName()
: DatasetName.Trim();
// Find all the members of this dataset.
var allFilesToCopy = DataSetManager.ListOfFilesInDataSetAsync(dataset, m => DisplayStatus($"Listing Files in {dataset}", m), failNow: () => Stopping).Result;
if (nFiles != 0)
{
allFilesToCopy = allFilesToCopy
.OrderBy(u => u.AbsolutePath)
.Take(nFiles)
.ToArray();
}
// If we have a location, we want to do a copy. If we don't have a location, then we just
// want to get the files somewhere local.
var loc = Location.AsIPlace().Result;
//WriteError(new ErrorRecord(err, "NoSuchLocation", ErrorCategory.InvalidArgument, null));
var resultUris = loc == null
? DataSetManager.MakeFilesLocalAsync(allFilesToCopy, m => DisplayStatus($"Downloading {dataset}", m), failNow : () => Stopping, timeout : Timeout).Result
: DataSetManager.CopyFilesToAsync(loc, allFilesToCopy, m => DisplayStatus($"Downloading {dataset}", m), failNow: () => Stopping, timeout: Timeout).Result;
// Dump all the returned files out to whatever is next in the pipeline.
if (PassThru.IsPresent)
{
using (var pl = listener.PauseListening())
{
foreach (var ds in resultUris)
{
WriteObject(ds);
}
}
}
}
finally
{
ClearStatus();
Trace.Listeners.Remove(listener);
}
}
public override void InitialiseFeatures(
DatasetName datasetName,
object[,] x,
object[] y)
{
Debug.Assert(x != null, "The feature vector must not be null.");
Debug.Assert(y != null, "The targe variable must not be null.");
Vector[] xV = null;
Range r = null;
// Validate
_validate.Dataset(
labels: _mapping.GetClassLabels().ToArray(),
datasetName: datasetName,
x: x,
y: y);
// Set meta data
_numFeatures = x.GetLength(0);
_numObservations = x.GetLength(1);
// Transpose
double[][] xTrans = new double[_numObservations][];
for (int j = 0; j < _numObservations; j++)
{
xTrans[j] = new double[_numFeatures];
for (int i = 0; i < _numFeatures; i++)
{
xTrans[j][i] = (double)x[i, j];
}
}
// Set target
_y = Variable.Observed(
Array.ConvertAll(y, v => (
System.Convert.ToInt64(v) > 0) ? true : false)).Named(
"y." + datasetName.ToString());
// Set features
xV = new Vector[_numObservations];
r = _y.Range.Named("person");
for (int i = 0; i < _numObservations; i++)
{
xV[i] = Vector.FromArray(xTrans[i]);
}
_x = Variable.Observed(xV, r).Named("x." + datasetName.ToString());
_availableDatasetName = datasetName;
}
public void RemoveFeature(DatasetName datasetName)
{
if (_datasets.ContainsKey(datasetName))
{
_datasets.Remove(datasetName);
// Reclaim the memory reserved
if (_datasets.Count == 0)
{
_datasets.Clear();
_datasets = null;
_datasets = new Dictionary <DatasetName, string[, ]>();
}
}
}
/// <summary>Snippet for ListTableSpecsAsync</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public async Task ListTableSpecsRequestObjectAsync()
{
// Create client
AutoMlClient autoMlClient = await AutoMlClient.CreateAsync();
// Initialize request argument(s)
ListTableSpecsRequest request = new ListTableSpecsRequest
{
ParentAsDatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
FieldMask = new FieldMask(),
Filter = "",
};
// Make the request
PagedAsyncEnumerable <ListTableSpecsResponse, TableSpec> response = autoMlClient.ListTableSpecsAsync(request);
// Iterate over all response items, lazily performing RPCs as required
await response.ForEachAsync((TableSpec item) =>
{
// Do something with each item
Console.WriteLine(item);
});
// Or iterate over pages (of server-defined size), performing one RPC per page
await response.AsRawResponses().ForEachAsync((ListTableSpecsResponse page) =>
{
// Do something with each page of items
Console.WriteLine("A page of results:");
foreach (TableSpec item in page)
{
// Do something with each item
Console.WriteLine(item);
}
});
// Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required
int pageSize = 10;
Page <TableSpec> singlePage = await response.ReadPageAsync(pageSize);
// Do something with the page of items
Console.WriteLine($"A page of {pageSize} results (unless it's the final page):");
foreach (TableSpec item in singlePage)
{
// Do something with each item
Console.WriteLine(item);
}
// Store the pageToken, for when the next page is required.
string nextPageToken = singlePage.NextPageToken;
}
/// <summary>
/// Get train and test datsets.
/// </summary>
public void ReadInputData(
string modelName,
string modelProcedure,
string modellingGroup,
string submodel,
string submodelCondValue,
DatasetName datasetName,
Dictionary <DatasetName, string> samplingsCombDatabase,
Dictionary <DatasetName, string> samplingsCombSample)
{
TraceListeners.Log(TraceEventType.Information, 0,
"ClassifierMain::ReadInputData...", false, true);
string query;
string[] featureNames;
// Read
query = "CALL " + Constants.MYSQL_DATABASES_MYSQL[samplingsCombDatabase[datasetName]]
+ "." + modelName + "(\"" + modelProcedure + "\", \"" + modellingGroup + "\", \""
+ submodel + "\"," + submodelCondValue + ",\"" + samplingsCombSample[datasetName] + "\");";
ReadMySQL reader = new ReadMySQL(
samplingsCombDatabase[datasetName],
Constants.MYSQL_SERVER,
Constants.MYSQL_USERNAME,
Constants.MYSQL_PASSWORD,
Constants.MYSQL_DATABASES_MYSQL[samplingsCombDatabase[datasetName]],
Constants.MYSQL_CMD_TIMEOUT,
Constants.MYSQL_CONN_TIMEOUT,
Constants.MYSQL_NULL_DEFAULT.ToString());
reader.Read(query);
// Set
featureNames = reader.GetColumnsNames();
_features.Set(
dataset: reader.GetColumns(featureNames),
datasetName: datasetName,
featuresByOrder: featureNames);
// Convert & Transform
_features.ConvertFeatures(
datasetName: datasetName,
standardConversion: true);
// Close
reader.CloseConnection();
}
private object Convert(
DatasetName datasetName,
string featureName,
int featureIndex,
int observationIndex,
string featureType = null)
{
object output = null;
if (featureType == null)
{
featureType = _featuresTypes[featureName];
}
switch (featureType.ToLower())
{
case "bool":
output = (System.Convert.ToInt64(
_datasets[datasetName][featureIndex, observationIndex]) > 0)
? true : false;
break;
case "double":
output = System.Convert.ToDouble(
_datasets[datasetName][featureIndex, observationIndex]);
break;
case "int":
output = System.Convert.ToInt32(
_datasets[datasetName][featureIndex, observationIndex]);
break;
case "date":
// break;
case "string":
output = _datasets[datasetName][featureIndex, observationIndex];
break;
default:
TraceListeners.Log(TraceEventType.Error, 0,
"Invalid variable type for " + featureName +
" in the metadata specification", true, true);
break;
}
return(output);
}
public override int GetHashCode()
{
int hash = 1;
if (Id.Length != 0)
{
hash ^= Id.GetHashCode();
}
if (Consumer.Length != 0)
{
hash ^= Consumer.GetHashCode();
}
if (DatasetName.Length != 0)
{
hash ^= DatasetName.GetHashCode();
}
if (DatasetType.Length != 0)
{
hash ^= DatasetType.GetHashCode();
}
if (DatasetId.Length != 0)
{
hash ^= DatasetId.GetHashCode();
}
hash ^= tags_.GetHashCode();
if (data_ != null)
{
hash ^= Data.GetHashCode();
}
hash ^= targetValues_.GetHashCode();
if (model_ != null)
{
hash ^= Model.GetHashCode();
}
hash ^= prediction_.GetHashCode();
if (TimeAdded != 0UL)
{
hash ^= TimeAdded.GetHashCode();
}
if (_unknownFields != null)
{
hash ^= _unknownFields.GetHashCode();
}
return(hash);
}
public override void InitialiseFeatures(
DatasetName datasetName,
object[,] x,
object[] y)
{
Debug.Assert(x != null, "The feature vector must not be null.");
Debug.Assert(y != null, "The targe variable must not be null.");
// Validate
_validate.Dataset(
labels: _mapping.GetClassLabels().ToArray(),
datasetName: datasetName,
x: x,
y: y);
// Set meta data
_numFeatures = x.GetLength(0);
_numObservations = x.GetLength(1);
// Transpose
double[][] xTrans = new double[_numObservations][];
for (int j = 0; j < _numObservations; j++)
{
xTrans[j] = new double[_numFeatures];
for (int i = 0; i < _numFeatures; i++)
{
xTrans[j][i] = (double)x[i, j];
}
}
// Set target
_y = new List <string>(
Array.ConvertAll(y, v => v.ToString()));
// Set features
_x = new Vector[_numObservations];
for (int i = 0; i < _numObservations; i++)
{
_x[i] = Vector.FromArray(xTrans[i]);
}
_availableDatasetName = datasetName;
}
public override int GetHashCode()
{
int hash = 1;
if (DatasetName.Length != 0)
{
hash ^= DatasetName.GetHashCode();
}
hash ^= inputs_.GetHashCode();
if (options_ != null)
{
hash ^= Options.GetHashCode();
}
if (_unknownFields != null)
{
hash ^= _unknownFields.GetHashCode();
}
return(hash);
}
/// <summary>Snippet for ListTableSpecs</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void ListTableSpecsResourceNames()
{
// Create client
AutoMlClient autoMlClient = AutoMlClient.Create();
// Initialize request argument(s)
DatasetName parent = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]");
// Make the request
PagedEnumerable <ListTableSpecsResponse, TableSpec> response = autoMlClient.ListTableSpecs(parent);
// Iterate over all response items, lazily performing RPCs as required
foreach (TableSpec item in response)
{
// Do something with each item
Console.WriteLine(item);
}
// Or iterate over pages (of server-defined size), performing one RPC per page
foreach (ListTableSpecsResponse page in response.AsRawResponses())
{
// Do something with each page of items
Console.WriteLine("A page of results:");
foreach (TableSpec item in page)
{
// Do something with each item
Console.WriteLine(item);
}
}
// Or retrieve a single page of known size (unless it's the final page), performing as many RPCs as required
int pageSize = 10;
Page <TableSpec> singlePage = response.ReadPage(pageSize);
// Do something with the page of items
Console.WriteLine($"A page of {pageSize} results (unless it's the final page):");
foreach (TableSpec item in singlePage)
{
// Do something with each item
Console.WriteLine(item);
}
// Store the pageToken, for when the next page is required.
string nextPageToken = singlePage.NextPageToken;
}
public void RemoveFeatureConverted(DatasetName datasetName)
{
if (_featuresConverted.ContainsKey(datasetName) &
_targetConverted.ContainsKey(datasetName))
{
_featuresConverted.Remove(datasetName);
_targetConverted.Remove(datasetName);
// Reclaim the memory reserved
if (_featuresConverted.Count == 0)
{
_featuresConverted.Clear();
_featuresConverted = null;
_featuresConverted = new Dictionary <DatasetName, object[, ]>();
_targetConverted.Clear();
_targetConverted = null;
_targetConverted = new Dictionary <DatasetName, object[]>();
}
}
}
public async stt::Task GetDatasetRequestObjectAsync()
{
moq::Mock <DatasetService.DatasetServiceClient> mockGrpcClient = new moq::Mock <DatasetService.DatasetServiceClient>(moq::MockBehavior.Strict);
mockGrpcClient.Setup(x => x.CreateOperationsClient()).Returns(new moq::Mock <lro::Operations.OperationsClient>().Object);
GetDatasetRequest request = new GetDatasetRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
ReadMask = new wkt::FieldMask(),
};
Dataset expectedResponse = new Dataset
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
DisplayName = "display_name137f65c2",
MetadataSchemaUri = "metadata_schema_uric874bf0a",
CreateTime = new wkt::Timestamp(),
UpdateTime = new wkt::Timestamp(),
Etag = "etage8ad7218",
Labels =
{
{
"key8a0b6e3c",
"value60c16320"
},
},
Metadata = new wkt::Value(),
EncryptionSpec = new EncryptionSpec(),
Description = "description2cf9da67",
};
mockGrpcClient.Setup(x => x.GetDatasetAsync(request, moq::It.IsAny <grpccore::CallOptions>())).Returns(new grpccore::AsyncUnaryCall <Dataset>(stt::Task.FromResult(expectedResponse), null, null, null, null));
DatasetServiceClient client = new DatasetServiceClientImpl(mockGrpcClient.Object, null);
Dataset responseCallSettings = await client.GetDatasetAsync(request, gaxgrpc::CallSettings.FromCancellationToken(st::CancellationToken.None));
xunit::Assert.Same(expectedResponse, responseCallSettings);
Dataset responseCancellationToken = await client.GetDatasetAsync(request, st::CancellationToken.None);
xunit::Assert.Same(expectedResponse, responseCancellationToken);
mockGrpcClient.VerifyAll();
}
public void Set(
Dictionary <string, List <string> > dataset,
DatasetName datasetName,
string[] featuresByOrder)
{
TraceListeners.Log(TraceEventType.Information, 0, "Features::Set...", false, true);
int numFeatures = dataset.Keys.Count;
int numObservations = dataset[dataset.Keys.First()].Count;
// Evaluate
if (_datasets.ContainsKey(datasetName))
{
TraceListeners.Log(TraceEventType.Warning, 0,
"Overwriting: " + datasetName.ToString(), false, true);
}
// Set
try
{
_featuresKey[datasetName] = new Dictionary <string, int>();
_datasets[datasetName] = new string[numFeatures, numObservations];
int i = 0;
foreach (string k in featuresByOrder)
{
_featuresKey[datasetName][k] = i;
for (int j = 0; j < numObservations; j++)
{
_datasets[datasetName][i, j] = dataset[k][j];
}
i++;
}
}
catch (Exception e)
{
TraceListeners.Log(TraceEventType.Error, 0, e.ToString(), true, true);
}
}
public BPMMapped(
string[] labels)
{
Debug.Assert(labels != null, "The labels must not be null.");
Debug.Assert(labels.Length == 2, "The labels must have two possible values.");
// Initialise the validations
_validate = new Validate();
// Create a BPM from the mapping
_mapping = new GenericClassifierMapping(labels);
_classifier = BayesPointMachineClassifier.CreateBinaryClassifier(_mapping);
// Evaluator mapping
var evaluatorMapping = _mapping.ForEvaluation();
_evaluator = new ClassifierEvaluator
<IList <Vector>, int, IList <string>, string>(evaluatorMapping);
// Other initialisations
_availableDatasetName = new DatasetName();
_numObservations = 0;
_numFeatures = 0;
}
/// <summary>
/// Returns a hash code for this instance.
/// </summary>
/// <returns>
/// A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table.
/// </returns>
public override int GetHashCode()
{
return(DatasetName.GetHashCode());
}
/// <summary>
/// Fetch a particular dataset.
/// </summary>
protected override void ProcessRecord()
{
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Get the actual dataset name.
var dataset = DatasetName.Trim();
if (ParameterSetName == "job")
{
// Get the job, see if it is finished, and then get the output dataset.
var job = JobParser.FindJob(JobName, JobVersion);
if (job == null)
{
throw new ArgumentException($"Job {JobName} v{JobVersion} is not known to the system!");
}
// Get the resulting job name for this guy.
var pandaJobName = job.ResultingDataSetName(JobSourceDatasetName, JobIteration) + "/";
// Now, to get the output dataset, we need to fetch the job.
var pandaTask = pandaJobName.FindPandaJobWithTaskName(true);
if (pandaTask == null)
{
throw new ArgumentException($"No panda task found with name '{pandaJobName}' for job '{JobName}' v{JobVersion}.");
}
var containers = pandaTask.DataSetNamesOUT();
if (containers.Length > 1)
{
throw new ArgumentException($"There are more than one output container for the panda task {pandaTask.jeditaskid} - can't decide. Need code upgrade!! Thanks for the fish!");
}
dataset = containers.First();
}
// Find all the members of this dataset.
var allFilesToCopy = DataSetManager.ListOfFilesInDataSetAsync(dataset, m => DisplayStatus($"Listing Files in {dataset}", m), failNow: () => Stopping)
.Result;
if (nFiles != 0)
{
allFilesToCopy = allFilesToCopy
.OrderBy(u => u.AbsolutePath)
.Take(nFiles)
.ToArray();
}
// Turn the source and destination locations into actual locations.
var locSource = SourceLocation.AsIPlace().Result;
var locDest = DestinationLocation.AsIPlace().Result;
// Do the actual copy. This will fail if one of the files can't be found at the source.
var resultUris = DataSetManager
.CopyFilesAsync(locSource, locDest, allFilesToCopy, mbox => DisplayStatus($"Downloading {dataset}", mbox), failNow: () => Stopping, timeout: Timeout)
.Result;
// Dump all the returned files out to whatever is next in the pipeline.
if (PassThru.IsPresent)
{
using (var pl = listener.PauseListening())
{
foreach (var ds in resultUris)
{
WriteObject(ds);
}
}
}
}
finally
{
Trace.Listeners.Remove(listener);
}
}
