public void queue_folder()
{
var sourceFolder = @"c:\Downloads\video\";
var taskFolder = @"c:\temp\dsqueue";
var files = Directory.GetFiles(sourceFolder, "*.*", SearchOption.AllDirectories);
var docs = new DocumentStoreServiceClient(new Uri("http://ds"), "docs");
// var demo = new DocumentStoreServiceClient(new Uri("http://ds"), "demo");
var counter = 1;
Parallel.ForEach(files, file =>
{
var handle = "import_" + counter++;
var taskDoc = docs.CreateDocumentImportData(
Guid.NewGuid(),
file,
Path.GetFileName(file),
new DocumentHandle(handle)
);
taskDoc.DeleteAfterImport = false;
var docsFile = Path.Combine(taskFolder, "doc_" + handle + "_" + DateTime.Now.Ticks);
docs.QueueDocumentImport(taskDoc, docsFile);
//var taskDemo = demo.CreateDocumentImportData(
// Guid.NewGuid(),
// file,
// new DocumentHandle(handle)
//);
//taskDemo.DeleteAfterImport = false;
//var demoFile = Path.Combine(taskFolder, "demo_" + handle + "_" + DateTime.Now.Ticks);
//demo.QueueDocumentImport(taskDemo, demoFile);
});
}
public void SetUp()
{
var config = new DocumentStoreTestConfiguration(_engineVersion);
MongoDbTestConnectionProvider.DropTestsTenant();
config.SetTestAddress(TestConfig.TestHostServiceAddress);
_documentStoreService = new DocumentStoreBootstrapper();
_documentStoreService.Start(config);
_documentStoreClient = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.Tenant
);
_tenant = ContainerAccessor.Instance.Resolve <TenantManager>().GetTenant(new TenantId(TestConfig.Tenant));
//Issue: https://github.com/ProximoSrl/Jarvis.DocumentStore/issues/26
//you need to resolve the IReader that in turns resolves the ProjectionEngine, becauase if you
//directly resolve the ITriggerProjectionsUpdate, projection engine will be resolved multiple times.
_tenant.Container.Resolve <IReader <StreamReadModel, Int64> >();
_projections = _tenant.Container.Resolve <ITriggerProjectionsUpdate>();
_documentDescriptorCollection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection <DocumentDescriptorReadModel>("rm.DocumentDescriptor");
_documentCollection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection <DocumentReadModel>("rm.Document");
_commitCollection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection <BsonDocument>("Commits");
_blobStore = _tenant.Container.Resolve <IBlobStore>();
MongoFlatMapper.EnableFlatMapping(true);
}
public void queue_folder()
{
var sourceFolder = @"c:\Downloads\video\";
var taskFolder = @"c:\temp\dsqueue";
var files = Directory.GetFiles(sourceFolder, "*.*", SearchOption.AllDirectories);
var docs = new DocumentStoreServiceClient(new Uri("http://ds"), "docs");
// var demo = new DocumentStoreServiceClient(new Uri("http://ds"), "demo");
var counter = 1;
Parallel.ForEach(files, file =>
{
var handle = "import_" + counter++;
var taskDoc = docs.CreateDocumentImportData(
Guid.NewGuid(),
file,
Path.GetFileName(file),
new DocumentHandle(handle)
);
taskDoc.DeleteAfterImport = false;
var docsFile = Path.Combine(taskFolder, "doc_" + handle + "_" + DateTime.Now.Ticks);
docs.QueueDocumentImport(taskDoc, docsFile);
//var taskDemo = demo.CreateDocumentImportData(
// Guid.NewGuid(),
// file,
// new DocumentHandle(handle)
//);
//taskDemo.DeleteAfterImport = false;
//var demoFile = Path.Combine(taskFolder, "demo_" + handle + "_" + DateTime.Now.Ticks);
//demo.QueueDocumentImport(taskDemo, demoFile);
});
}
static int Run()
{
Console.WriteLine("------------------------------------------");
Console.WriteLine("Running");
Console.WriteLine("\tServer: {0}", _endPoint.AbsoluteUri);
Console.WriteLine("\tFolder: {0}", _folder);
Console.WriteLine("------------------------------------------");
var client = new DocumentStoreServiceClient(_endPoint);
int counter = 1;
Parallel.ForEach(Directory.GetFiles(_folder), file =>
{
Console.WriteLine("Processing file {0}", file);
var id = Interlocked.Increment(ref counter);
try
{
client.UploadAsync(file, "File_" + id).Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.InnerExceptions)
{
Console.WriteLine("Error: {0}", ex.Message);
}
}
});
Console.WriteLine("------------------------------------------");
return(0);
}
static int Run()
{
Console.WriteLine("------------------------------------------");
Console.WriteLine("Running");
Console.WriteLine("\tServer: {0}", _endPoint.AbsoluteUri);
Console.WriteLine("\tFolder: {0}", _folder);
Console.WriteLine("------------------------------------------");
var client = new DocumentStoreServiceClient(_endPoint);
int counter = 1;
Parallel.ForEach(Directory.GetFiles(_folder), file =>
{
Console.WriteLine("Processing file {0}", file);
var id = Interlocked.Increment(ref counter);
try
{
client.UploadAsync(file, "File_" + id).Wait();
}
catch (AggregateException aex)
{
foreach(var ex in aex.InnerExceptions)
Console.WriteLine("Error: {0}", ex.Message);
}
});
Console.WriteLine("------------------------------------------");
return 0;
}
public void OneTimeSetUp()
{
_docs = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.DocsTenant
);
}
protected async Task <Boolean> AddFormatToDocumentFromFile(
string tenantId,
String jobId,
Client.Model.DocumentFormat format,
string pathToFile,
IDictionary <string, object> customData)
{
DocumentStoreServiceClient client = GetDocumentStoreClient(tenantId);
AddFormatFromFileToDocumentModel model = new AddFormatFromFileToDocumentModel
{
CreatedById = this.PipelineId,
JobId = jobId,
QueueName = this.QueueName,
Format = format,
PathToFile = pathToFile
};
var response = await client.AddFormatToDocument(model, customData).ConfigureAwait(false);
if (Logger.IsInfoEnabled)
{
Logger.Info($"Job {this.GetType()} Added format {format} to handle with job id {jobId}");
}
return(response != null);
}
protected Task <Stream> GetBlobFormatReader(
String tenantId,
String jobId)
{
DocumentStoreServiceClient client = GetDocumentStoreClient(tenantId);
var reader = client.OpenBlobIdForRead(QueueName, jobId);
return(reader.OpenStream());
}
private async Task <string> GetfileNameFromHandle(DocumentStoreServiceClient client, DocumentHandle handle)
{
try
{
return(await client.GetFileNameAsync(handle));
}
catch (Exception)
{
return(String.Empty);
}
}
public void TestFixtureSetUp()
{
_docs = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.DocsTenant
);
_demo = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.DemoTenant
);
}
public void TestFixtureSetUp()
{
_docs = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.DocsTenant
);
_demo = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.DemoTenant
);
}
public void should_create_a_valid_document_import_data()
{
var client = new DocumentStoreServiceClient(new Uri("http://ds"), "test");
var did = client.CreateDocumentImportData(TaskId, "c:\\temp\\a file.docx", "a file.docx", Doc);
Assert.AreEqual("file:///c:/temp/a%20file.docx", did.Uri.AbsoluteUri);
Assert.AreEqual("test", did.Tenant);
Assert.AreEqual("a file.docx", did.FileName);
Assert.AreEqual(Doc, did.Handle);
Assert.IsFalse(did.DeleteAfterImport);
Assert.AreEqual(DocumentStoreServiceClient.OriginalFormat, did.Format);
}
public void should_create_a_valid_document_import_data()
{
var client = new DocumentStoreServiceClient(new Uri("http://ds"), "test");
var did = client.CreateDocumentImportData(TaskId, "c:\\temp\\a file.docx","a file.docx", Doc);
Assert.AreEqual("file:///c:/temp/a%20file.docx", did.Uri.AbsoluteUri);
Assert.AreEqual("test", did.Tenant);
Assert.AreEqual("a file.docx", did.FileName);
Assert.AreEqual(Doc, did.Handle);
Assert.IsFalse(did.DeleteAfterImport);
Assert.AreEqual(DocumentStoreServiceClient.OriginalFormat, did.Format);
}
protected async Task <Boolean> AddAttachmentToHandle(
string tenantId,
String jobId,
string pathToFile,
String source,
String relativePath,
IDictionary <string, object> customData)
{
DocumentStoreServiceClient client = GetDocumentStoreClient(tenantId);
var response = await client.UploadAttachmentAsync(pathToFile, this.QueueName, jobId, source, relativePath, customData).ConfigureAwait(false);
return(response != null);
}
private static async Task <bool> InnerGetPdf(string workingFolder, DocumentStoreServiceClient client, List <FileToComposeData> files, string handle, DocumentHandle documentHandle, bool pdfExists)
{
var pdfData = client.OpenRead(documentHandle, DocumentFormats.Pdf);
var tempFile = Path.Combine(workingFolder, Guid.NewGuid() + ".pdf");
using (var downloaded = new FileStream(tempFile, FileMode.OpenOrCreate, FileAccess.Write))
{
var stream = await pdfData.OpenStream();
stream.CopyTo(downloaded);
}
files.Add(FileToComposeData.FromDownloadedPdf(tempFile, handle));
pdfExists = true;
return(pdfExists);
}
public void SetUp()
{
var config = new DocumentStoreTestConfiguration(_engineVersion);
MongoDbTestConnectionProvider.DropTestsTenant();
config.SetTestAddress(TestConfig.ServerAddress);
_documentStoreService = new DocumentStoreBootstrapper();
_documentStoreService.Start(config);
_documentStoreClient = new DocumentStoreServiceClient(
TestConfig.ServerAddress,
TestConfig.Tenant
);
_tenant = ContainerAccessor.Instance.Resolve<TenantManager>().Current;
_projections = _tenant.Container.Resolve<ITriggerProjectionsUpdate>();
_documentDescriptorCollection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection<DocumentDescriptorReadModel>("rm.DocumentDescriptor");
_documentCollection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection<DocumentReadModel>("rm.Document");
_commitCollection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection<BsonDocument>("Commits");
_blobStore = _tenant.Container.Resolve<IBlobStore>();
MongoFlatMapper.EnableFlatMapping(true);
}
protected async Task <Boolean> AddFormatToDocumentFromObject(string tenantId,
String queueName,
String jobId,
Client.Model.DocumentFormat format,
Object obj,
String originalFileName,
IDictionary <string, object> customData)
{
DocumentStoreServiceClient client = GetDocumentStoreClient(tenantId);
AddFormatFromObjectToDocumentModel model = new AddFormatFromObjectToDocumentModel
{
CreatedById = this.PipelineId,
JobId = jobId,
QueueName = queueName,
Format = format,
FileName = originalFileName,
StringContent = JsonConvert.SerializeObject(obj),
};
var response = await client.AddFormatToDocument(model, customData).ConfigureAwait(false);
return(response != null);
}
public void should_serialize_document_import_data()
{
var fname = Path.Combine(TestConfig.TempFolder, "a_file_to_import");
var client = new DocumentStoreServiceClient(new Uri("http://ds"), "test");
var did = client.CreateDocumentImportData(TaskId, "c:\\temp\\a file.docx", "a file.docx", Doc);
client.QueueDocumentImport(did, fname);
Assert.IsTrue(File.Exists(fname + ".dsimport"));
const string expected =
@"{
""TaskId"": ""9a29d730-f57a-41e4-92ba-55b7d99712a2"",
""Uri"": ""c:\\temp\\a file.docx"",
""FileName"": ""a file.docx"",
""Handle"": ""doc"",
""Format"": ""original"",
""Tenant"": ""test"",
""CustomData"": null,
""DeleteAfterImport"": false
}";
Assert.AreEqual(expected, File.ReadAllText(fname + ".dsimport"));
}
protected async Task <String> DownloadBlob(
String tenantId,
String jobId,
String originalFileName,
String workingFolder)
{
String fileName = Path.Combine(workingFolder, originalFileName);
//TooLongNames should be truncated, tika, or other libraries are not able to access
//too long file names. Max file name is 260, but some libraries or task can append some more char
//ex tika.html
fileName = SanitizeFileNameForLength(fileName);
DocumentStoreServiceClient client = GetDocumentStoreClient(tenantId);
var reader = client.OpenBlobIdForRead(this.QueueName, jobId);
using (var downloaded = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.Write))
{
var stream = await reader.OpenStream().ConfigureAwait(false);
stream.CopyTo(downloaded);
}
Logger.DebugFormat("Downloaded blob for job {0} for tenant {1} in local file {2}", jobId, tenantId, fileName);
return(fileName);
}
public void should_serialize_document_import_data()
{
var fname = Path.Combine(TestConfig.TempFolder, "a_file_to_import");
var client = new DocumentStoreServiceClient(new Uri("http://ds"), "test");
var did = client.CreateDocumentImportData(TaskId, "c:\\temp\\a file.docx", "a file.docx", Doc);
client.QueueDocumentImport(did, fname);
Assert.IsTrue(File.Exists(fname + ".dsimport"));
const string expected =
@"{
""TaskId"": ""9a29d730-f57a-41e4-92ba-55b7d99712a2"",
""Uri"": ""c:\\temp\\a file.docx"",
""FileName"": ""a file.docx"",
""Handle"": ""doc"",
""Format"": ""original"",
""Tenant"": ""test"",
""CustomData"": null,
""DeleteAfterImport"": false
}";
Assert.AreEqual(expected, File.ReadAllText(fname + ".dsimport"));
}
private async Task<string> GetfileNameFromHandle(DocumentStoreServiceClient client, DocumentHandle handle)
{
try
{
return await client.GetFileNameAsync(handle);
}
catch (Exception)
{
return String.Empty;
}
}
