public async void CreateDatasetRequiresNonNullDatasetInfo()
{
var client = new Mock <IElasticClient>();
AssertIndexExists(client, _indexName);
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
await Assert.ThrowsAsync <ArgumentNullException>(() => datasetStore.CreateOrUpdateDatasetRecordAsync(null));
}
private async Task ImportDatasetsAsync()
{
var datasetJson = File.ReadAllText(_options.DatasetsJsonFile);
var datasets = JsonConvert.DeserializeObject <List <LegacyDatasetInfo> >(datasetJson);
foreach (var ds in datasets)
{
await _datasetStore.CreateOrUpdateDatasetRecordAsync(new DatasetInfo
{
OwnerId = ds.OwnerId,
RepositoryId = FixRepositoryId(ds.RepositoryId),
DatasetId = ds.DatasetId,
LastModified = ds.LastModified,
ShowOnHomePage = ds.ShowOnHomePage,
CsvwMetadata = ds.Metadata,
Tags = ds.Metadata["dcat:keyword"]?.ToObject <List <string> >()
});
}
}
public async void CreateDatasetThrowsWhenInsertFails()
{
var mockResponse = new Mock <IIndexResponse>();
mockResponse.SetupGet(x => x.IsValid).Returns(false);
var client = new Mock <IElasticClient>();
AssertIndexExists(client, _indexName);
client.Setup(x => x.IndexDocumentAsync <DatasetInfo>(It.IsAny <DatasetInfo>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(mockResponse.Object).Verifiable();
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
var emptyDatasetInfo = new DatasetInfo();
await Assert.ThrowsAsync <DatasetStoreException>(() => datasetStore.CreateOrUpdateDatasetRecordAsync(emptyDatasetInfo));
client.Verify();
}
public async Task ItCanCreateAndRetrieveDataset()
{
var csvwJson = new JObject(new JProperty("dc:title", "Test Dataset"), new JProperty("dcat:keyword", new JArray("one", "two", "three")));
var voidJson = new JObject(
new JProperty("void:triples", "100"),
new JProperty("void:dataDump", new JArray("https://github.com/jennet/animated-barnacle/releases/download/acsv_csv_20180207_170200/acsv_csv_20180207_170200.nt.gz", "http://datadock.io/jennet/animated-barnacle/csv/acsv.csv/acsv.csv")));
var datasetInfo = new DatasetInfo
{
OwnerId = "owner-1",
RepositoryId = "repo-1",
DatasetId = "test.csv",
ShowOnHomePage = true,
LastModified = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)),
CsvwMetadata = csvwJson,
VoidMetadata = voidJson
};
await _store.CreateOrUpdateDatasetRecordAsync(datasetInfo);
//Thread.Sleep(1000);
var retrievedDataset = await _store.GetDatasetInfoAsync("owner-1", "repo-1", "test.csv");
retrievedDataset.Id.Should().Be($"owner-1/repo-1/test.csv");
((string)retrievedDataset.OwnerId).Should().Be("owner-1");
((string)retrievedDataset.RepositoryId).Should().Be("repo-1");
((string)retrievedDataset.DatasetId).Should().Be("test.csv");
retrievedDataset.LastModified.Should().BeCloseTo(datasetInfo.LastModified);
var retrievedByIdDataset = await _store.GetDatasetInfoByIdAsync("owner-1/repo-1/test.csv");
retrievedByIdDataset.Id.Should().Be($"owner-1/repo-1/test.csv");
((string)retrievedByIdDataset.OwnerId).Should().Be("owner-1");
((string)retrievedByIdDataset.RepositoryId).Should().Be("repo-1");
((string)retrievedByIdDataset.DatasetId).Should().Be("test.csv");
retrievedByIdDataset.LastModified.Should().BeCloseTo(datasetInfo.LastModified);
var retrievedVoid = retrievedByIdDataset.VoidMetadata;
var tripleCount = retrievedVoid["void:triples"].Value <string>();
Assert.Equal("100", tripleCount);
}
public async void CreateDatasetInsertsIntoDatasetsIndex()
{
var mockResponse = new Mock <IIndexResponse>();
mockResponse.SetupGet(x => x.IsValid).Returns(true);
var client = new Mock <IElasticClient>();
AssertIndexExists(client, _indexName);
client.Setup(x => x.IndexDocumentAsync <DatasetInfo>(It.IsAny <DatasetInfo>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(mockResponse.Object).Verifiable();
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
var csvwJson = new JObject(new JProperty("dc:title", "Test Dataset"), new JProperty("dcat:keyword", new JArray("one", "two", "three")));
var voidJson = new JObject(
new JProperty("void:triples", "100"),
new JProperty("void:dataDump", new JArray("https://github.com/jennet/animated-barnacle/releases/download/acsv_csv_20180207_170200/acsv_csv_20180207_170200.nt.gz", "http://datadock.io/jennet/animated-barnacle/csv/acsv.csv/acsv.csv")));
var datasetInfo = new DatasetInfo
{
OwnerId = "git-user",
RepositoryId = "repo-name",
DatasetId = "test.csv",
ShowOnHomePage = true,
LastModified = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)),
CsvwMetadata = csvwJson,
VoidMetadata = voidJson
};
Assert.Null(datasetInfo.Id);
var created = await datasetStore.CreateOrUpdateDatasetRecordAsync(datasetInfo);
client.Verify();
Assert.NotNull(created);
Assert.Equal($"{datasetInfo.OwnerId}/{datasetInfo.RepositoryId}/{datasetInfo.DatasetId}", datasetInfo.Id);
}