public void TestExportWhileWriting()
{
var storeName = Guid.NewGuid().ToString();
var client = GetClient();
client.CreateStore(storeName);
var batch1 = MakeTriples(0, 50000);
var batch2 = MakeTriples(50000, 51000);
var batch3 = MakeTriples(51000, 52000);
var batch4 = MakeTriples(52000, 53000);
// Verify batch size
var p = new NTriplesParser();
var counterSink = new CounterTripleSink();
p.Parse(new StringReader(batch1), counterSink, Constants.DefaultGraphUri);
Assert.AreEqual(50000, counterSink.Count);
var jobInfo = client.ExecuteTransaction(storeName, String.Empty, String.Empty, batch1);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
// Second export with parallel store writes
var exportJobInfo = client.StartExport(storeName, storeName + "_export.nt");
jobInfo = client.ExecuteTransaction(storeName, null, null, batch2);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
Assert.IsTrue(exportJobInfo.JobStarted, "Test inconclusive - export job completed before end of first concurrent import job."); // This is just to check that the export is still running while at least one commit occurs
jobInfo = client.ExecuteTransaction(storeName, null, null, batch3);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
jobInfo = client.ExecuteTransaction(storeName, null, null, batch4);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
while (!exportJobInfo.JobCompletedOk)
{
Assert.IsFalse(exportJobInfo.JobCompletedWithErrors);
Thread.Sleep(1000);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
}
var exportFile = new FileInfo("c:\\brightstar\\import\\" + storeName + "_export.nt");
Assert.IsTrue(exportFile.Exists);
var lineCount = File.ReadAllLines(exportFile.FullName).Where(x => !String.IsNullOrEmpty(x)).Count();
Assert.AreEqual(50000, lineCount);
}
public void TestExportWhileWriting()
{
const int firstBatchSize = 50000;
var storeName = Guid.NewGuid().ToString();
var client = GetClient();
client.CreateStore(storeName);
var batch1 = MakeTriples(0, firstBatchSize);
var batch2 = MakeTriples(firstBatchSize, firstBatchSize+1000);
var batch3 = MakeTriples(firstBatchSize+1000, firstBatchSize+2000);
var batch4 = MakeTriples(firstBatchSize+2000, firstBatchSize+3000);
// Verify batch size
var p = new NTriplesParser();
var counterSink = new CounterTripleSink();
p.Parse(new StringReader(batch1), counterSink, Constants.DefaultGraphUri);
Assert.AreEqual(firstBatchSize, counterSink.Count);
var jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData {InsertData = batch1});
Assert.AreEqual(true, jobInfo.JobCompletedOk);
// Second export with parallel store writes
var exportJobInfo = client.StartExport(storeName, storeName + "_export.nt", label:"Export Data");
Assert.That(exportJobInfo.Label, Is.EqualTo("Export Data"));
jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData{InsertData = batch2});
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
if (exportJobInfo.JobCompletedWithErrors)
{
Assert.Fail("Export job completed with errors: {0} : {1}", exportJobInfo.StatusMessage, exportJobInfo.ExceptionInfo);
}
if (exportJobInfo.JobCompletedOk)
{
Assert.Inconclusive("Export job completed before end of first concurrent import job.");
}
Assert.That(exportJobInfo.Label, Is.EqualTo("Export Data"));
jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData{InsertData= batch3});
Assert.AreEqual(true, jobInfo.JobCompletedOk);
jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData{InsertData = batch4});
Assert.AreEqual(true, jobInfo.JobCompletedOk);
while (!exportJobInfo.JobCompletedOk)
{
Assert.IsFalse(exportJobInfo.JobCompletedWithErrors);
Thread.Sleep(1000);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
}
var exportFile = new FileInfo(Path.Combine(Configuration.StoreLocation, "import", storeName + "_export.nt"));
Assert.IsTrue(exportFile.Exists);
var lineCount = File.ReadAllLines(exportFile.FullName).Count(x => !String.IsNullOrEmpty(x));
Assert.AreEqual(firstBatchSize, lineCount);
}
public void TestExportWhileWriting()
{
const int firstBatchSize = 50000;
var storeName = Guid.NewGuid().ToString();
var client = GetClient();
client.CreateStore(storeName);
var batch1 = MakeTriples(0, firstBatchSize);
var batch2 = MakeTriples(firstBatchSize, firstBatchSize+1000);
var batch3 = MakeTriples(firstBatchSize+1000, firstBatchSize+2000);
var batch4 = MakeTriples(firstBatchSize+2000, firstBatchSize+3000);
// Verify batch size
var p = new NTriplesParser();
var counterSink = new CounterTripleSink();
p.Parse(new StringReader(batch1), counterSink, Constants.DefaultGraphUri);
Assert.AreEqual(firstBatchSize, counterSink.Count);
var jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData
{
ExistencePreconditions = String.Empty,
DeletePatterns = String.Empty,
InsertData = batch1
});
jobInfo = WaitForJob(jobInfo, client, storeName);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
// Second export with parallel store writes
var exportJobInfo = client.StartExport(storeName, storeName + "_export.nt");
jobInfo = client.ExecuteTransaction(storeName,
new UpdateTransactionData
{
ExistencePreconditions = null,
DeletePatterns = null,
InsertData = batch2
});
jobInfo = WaitForJob(jobInfo, client, storeName);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
if (exportJobInfo.JobCompletedWithErrors)
{
Assert.Fail("Export job completed with errors: {0} : {1}", exportJobInfo.StatusMessage, exportJobInfo.ExceptionInfo);
}
if (exportJobInfo.JobCompletedOk)
{
Assert.Inconclusive("Export job completed before end of first concurrent import job.");
}
jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData{InsertData= batch3});
jobInfo = WaitForJob(jobInfo, client, storeName);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
jobInfo = client.ExecuteTransaction(storeName, new UpdateTransactionData{InsertData = batch4});
jobInfo = WaitForJob(jobInfo, client, storeName);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = WaitForJob(exportJobInfo, client, storeName);
Assert.That(exportJobInfo.JobCompletedOk);
var exportFile = new FileInfo("c:\\brightstar\\import\\" + storeName + "_export.nt");
Assert.IsTrue(exportFile.Exists);
var lineCount = File.ReadAllLines(exportFile.FullName).Count(x => !String.IsNullOrEmpty(x));
Assert.AreEqual(firstBatchSize, lineCount);
}
public void TestExportWhileWriting()
{
var storeName = Guid.NewGuid().ToString();
var client = GetClient();
client.CreateStore(storeName);
var batch1 = MakeTriples(0, 50000);
var batch2 = MakeTriples(50000, 51000);
var batch3 = MakeTriples(51000, 52000);
var batch4 = MakeTriples(52000, 53000);
// Verify batch size
var p = new NTriplesParser();
var counterSink = new CounterTripleSink();
p.Parse(new StringReader(batch1), counterSink, Constants.DefaultGraphUri);
Assert.AreEqual(50000, counterSink.Count);
var jobInfo = client.ExecuteTransaction(storeName, String.Empty, String.Empty, batch1);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
// Second export with parallel store writes
var exportJobInfo = client.StartExport(storeName, storeName + "_export.nt");
jobInfo = client.ExecuteTransaction(storeName, null, null, batch2);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
Assert.IsTrue(exportJobInfo.JobStarted, "Test inconclusive - export job completed before end of first concurrent import job."); // This is just to check that the export is still running while at least one commit occurs
jobInfo = client.ExecuteTransaction(storeName, null, null, batch3);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
jobInfo = client.ExecuteTransaction(storeName, null, null, batch4);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
while (!exportJobInfo.JobCompletedOk)
{
Assert.IsFalse(exportJobInfo.JobCompletedWithErrors);
Thread.Sleep(1000);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
}
var exportFile = new FileInfo("c:\\brightstar\\import\\" + storeName + "_export.nt");
Assert.IsTrue(exportFile.Exists);
var lineCount = File.ReadAllLines(exportFile.FullName).Where(x => !String.IsNullOrEmpty(x)).Count();
Assert.AreEqual(50000, lineCount);
}
public void TestExportWhileWriting()
{
int firstBatchSize = 50000;
var storeName = Guid.NewGuid().ToString();
var client = GetClient();
client.CreateStore(storeName);
var batch1 = MakeTriples(0, firstBatchSize);
var batch2 = MakeTriples(firstBatchSize, firstBatchSize+1000);
var batch3 = MakeTriples(firstBatchSize+1000, firstBatchSize+2000);
var batch4 = MakeTriples(firstBatchSize+2000, firstBatchSize+3000);
// Verify batch size
var p = new NTriplesParser();
var counterSink = new CounterTripleSink();
p.Parse(new StringReader(batch1), counterSink, Constants.DefaultGraphUri);
Assert.AreEqual(firstBatchSize, counterSink.Count);
var jobInfo = client.ExecuteTransaction(storeName, String.Empty, String.Empty, batch1);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
// Second export with parallel store writes
var exportJobInfo = client.StartExport(storeName, storeName + "_export.nt");
jobInfo = client.ExecuteTransaction(storeName, null, null, batch2);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
if (exportJobInfo.JobCompletedWithErrors)
{
Assert.Fail("Export job completed with errors: {0} : {1}", exportJobInfo.StatusMessage, exportJobInfo.ExceptionInfo);
}
if (exportJobInfo.JobCompletedOk)
{
Assert.Inconclusive("Export job completed before end of first concurrent import job.");
}
jobInfo = client.ExecuteTransaction(storeName, null, null, batch3);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
jobInfo = client.ExecuteTransaction(storeName, null, null, batch4);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
while (!exportJobInfo.JobCompletedOk)
{
Assert.IsFalse(exportJobInfo.JobCompletedWithErrors);
Thread.Sleep(1000);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
}
FileInfo exportFile;
if (client is BrightstarRestClient)
{
exportFile = new FileInfo("c:\\brightstar\\import\\" + storeName + "_export.nt");
}
else
{
exportFile = new FileInfo("brightstar\\import\\" + storeName + "_export.nt");
}
Assert.IsTrue(exportFile.Exists);
var lineCount = File.ReadAllLines(exportFile.FullName).Where(x => !String.IsNullOrEmpty(x)).Count();
Assert.AreEqual(firstBatchSize, lineCount);
}
public void TestExportWhileWriting()
{
int firstBatchSize = 50000;
var storeName = Guid.NewGuid().ToString();
var client = GetClient();
client.CreateStore(storeName);
var batch1 = MakeTriples(0, firstBatchSize);
var batch2 = MakeTriples(firstBatchSize, firstBatchSize + 1000);
var batch3 = MakeTriples(firstBatchSize + 1000, firstBatchSize + 2000);
var batch4 = MakeTriples(firstBatchSize + 2000, firstBatchSize + 3000);
// Verify batch size
var p = new NTriplesParser();
var counterSink = new CounterTripleSink();
p.Parse(new StringReader(batch1), counterSink, Constants.DefaultGraphUri);
Assert.AreEqual(firstBatchSize, counterSink.Count);
var jobInfo = client.ExecuteTransaction(storeName, String.Empty, String.Empty, batch1);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
// Second export with parallel store writes
var exportJobInfo = client.StartExport(storeName, storeName + "_export.nt");
jobInfo = client.ExecuteTransaction(storeName, null, null, batch2);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
if (exportJobInfo.JobCompletedWithErrors)
{
Assert.Fail("Export job completed with errors: {0} : {1}", exportJobInfo.StatusMessage, exportJobInfo.ExceptionInfo);
}
if (exportJobInfo.JobCompletedOk)
{
Assert.Inconclusive("Export job completed before end of first concurrent import job.");
}
jobInfo = client.ExecuteTransaction(storeName, null, null, batch3);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
jobInfo = client.ExecuteTransaction(storeName, null, null, batch4);
Assert.AreEqual(true, jobInfo.JobCompletedOk);
while (!exportJobInfo.JobCompletedOk)
{
Assert.IsFalse(exportJobInfo.JobCompletedWithErrors);
Thread.Sleep(1000);
exportJobInfo = client.GetJobInfo(storeName, exportJobInfo.JobId);
}
FileInfo exportFile;
if (client is BrightstarRestClient)
{
exportFile = new FileInfo("c:\\brightstar\\import\\" + storeName + "_export.nt");
}
else
{
exportFile = new FileInfo("brightstar\\import\\" + storeName + "_export.nt");
}
Assert.IsTrue(exportFile.Exists);
var lineCount = File.ReadAllLines(exportFile.FullName).Where(x => !String.IsNullOrEmpty(x)).Count();
Assert.AreEqual(firstBatchSize, lineCount);
}
