static void Main(string[] args)
{
Options options;
try
{
options = ValidateArguments(args);
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.Message);
Usage();
return;
}
try
{
var esClient = new ElasticClient(
new ConnectionSettings(
new SingleNodeConnectionPool(new Uri(options.ElasticsearchUrl)),
JsonNetSerializer.Default));
var config = new ApplicationConfiguration {
ElasticsearchUrl = options.ElasticsearchUrl
};
var datasetsStore = new DatasetStore(esClient, config);
var schemaStore = new SchemaStore(esClient, config);
var importer = new Importer(options, datasetsStore, schemaStore);
importer.RunAsync().Wait();
}
catch (Exception ex)
{
Console.Error.WriteLine("Error running import process: " + ex);
}
}
public async void CreateDatasetRequiresNonNullDatasetInfo()
{
var client = new Mock <IElasticClient>();
AssertIndexExists(client, _indexName);
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
await Assert.ThrowsAsync <ArgumentNullException>(() => datasetStore.CreateOrUpdateDatasetRecordAsync(null));
}
public void GetEntities_ValidCall_ReturnList <EntityStoreItem>()
{
//Arrange
DatasetStore datasetStore = new DatasetStore();
//Act
var datasets = datasetStore.GetEntities();
//Assert
Assert.IsNotNull(datasets);
Assert.IsNotEmpty(datasets);
Assert.That(datasets.Count, Is.EqualTo(_numberOfDatasets));
}
public async void CreateDatasetThrowsWhenInsertFails()
{
var mockResponse = new Mock <IIndexResponse>();
mockResponse.SetupGet(x => x.IsValid).Returns(false);
var client = new Mock <IElasticClient>();
AssertIndexExists(client, _indexName);
client.Setup(x => x.IndexDocumentAsync <DatasetInfo>(It.IsAny <DatasetInfo>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(mockResponse.Object).Verifiable();
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
var emptyDatasetInfo = new DatasetInfo();
await Assert.ThrowsAsync <DatasetStoreException>(() => datasetStore.CreateOrUpdateDatasetRecordAsync(emptyDatasetInfo));
client.Verify();
}
public async void CreateDatasetInsertsIntoDatasetsIndex()
{
var mockResponse = new Mock <IIndexResponse>();
mockResponse.SetupGet(x => x.IsValid).Returns(true);
var client = new Mock <IElasticClient>();
AssertIndexExists(client, _indexName);
client.Setup(x => x.IndexDocumentAsync <DatasetInfo>(It.IsAny <DatasetInfo>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(mockResponse.Object).Verifiable();
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
var csvwJson = new JObject(new JProperty("dc:title", "Test Dataset"), new JProperty("dcat:keyword", new JArray("one", "two", "three")));
var voidJson = new JObject(
new JProperty("void:triples", "100"),
new JProperty("void:dataDump", new JArray("https://github.com/jennet/animated-barnacle/releases/download/acsv_csv_20180207_170200/acsv_csv_20180207_170200.nt.gz", "http://datadock.io/jennet/animated-barnacle/csv/acsv.csv/acsv.csv")));
var datasetInfo = new DatasetInfo
{
OwnerId = "git-user",
RepositoryId = "repo-name",
DatasetId = "test.csv",
ShowOnHomePage = true,
LastModified = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(1)),
CsvwMetadata = csvwJson,
VoidMetadata = voidJson
};
Assert.Null(datasetInfo.Id);
var created = await datasetStore.CreateOrUpdateDatasetRecordAsync(datasetInfo);
client.Verify();
Assert.NotNull(created);
Assert.Equal($"{datasetInfo.OwnerId}/{datasetInfo.RepositoryId}/{datasetInfo.DatasetId}", datasetInfo.Id);
}
public void StoreCreatesIndexIfItDoesNotExist()
{
var client = new Mock <IElasticClient>();
var notExists = new Mock <IExistsResponse>();
notExists.SetupGet(x => x.Exists).Returns(false);
var indexCreated = new Mock <ICreateIndexResponse>();
indexCreated.SetupGet(x => x.Acknowledged).Returns(true);
client.Setup(x => x.IndexExists(It.IsAny <Indices>(), null)).Returns(notExists.Object);
client.Setup(x => x.CreateIndex(_indexName, It.IsAny <Func <CreateIndexDescriptor, ICreateIndexRequest> >()))
.Returns(indexCreated.Object).Verifiable();
var indexDict = new FluentDictionary <Type, string>();
var connectionSettings = new Mock <IConnectionSettingsValues>();
connectionSettings.Setup(x => x.DefaultIndices).Returns(indexDict).Verifiable();
client.SetupGet(x => x.ConnectionSettings).Returns(connectionSettings.Object).Verifiable();
var datasetStore = new DatasetStore(client.Object, new ApplicationConfiguration {
DatasetIndexName = _indexName
});
client.Verify();
}
public DatasetStoreFixture() : base()
{
Store = new DatasetStore(Client, Configuration);
InitializeStore().Wait();
Thread.Sleep(1000);
}
public DatasetStoreTests(ElasticsearchFixture fixture)
{
_fixture = fixture;
_store = new DatasetStore(fixture.Client, fixture.Configuration);
}
public DatasetStoreSearchTests(DatasetStoreFixture fixture)
{
_store = fixture.Store;
}
public Importer(Options opts, DatasetStore datasetStore, SchemaStore schemaStore)
{
_options = opts;
_datasetStore = datasetStore;
_schemaStore = schemaStore;
}