/// <summary>
/// Mounts the bucket. Mount is necessary before using a bucket to ensure the existence of tables and indexes.
/// </summary>
public async Task MountAsync(CancellationToken cancelToken = default)
{
if (this.Mounted)
{
return;
}
var filesTableResult = await EnsureTable(this.fileTableName, cancelToken)
.ConfigureAwait(false);
if (filesTableResult.TablesCreated == 1)
{
//index the file paths of completed files and status
ReqlFunction1 pathIx = row => { return(R.Array(row[FileInfo.StatusJsonName], row[FileInfo.FileNameJsonName], row[FileInfo.FinishedDateJsonName])); };
await CreateIndex(this.fileTableName, this.fileIndex, pathIx, cancelToken)
.ConfigureAwait(false);
//prefix IX
ReqlFunction1 prefixIx = doc =>
{
//return r.array(doc[FileInfo.FileNameJsonName].split("/").slice(1, -1), doc[FileInfo.FinishedDateJsonName]);
return(R.Branch(doc[FileInfo.StatusJsonName].Eq(Status.Completed),
R.Array(doc[FileInfo.FileNameJsonName].Split("/").Slice(1, -1), doc[FileInfo.FinishedDateJsonName]),
R.Error()));
};
await CreateIndex(this.fileTableName, this.fileIndexPrefix, prefixIx, cancelToken)
.ConfigureAwait(false);
}
// CHUNK TAABLE INDEXES
var chunkTableResult = await EnsureTable(this.chunkTableName, cancelToken)
.ConfigureAwait(false);
if (chunkTableResult.TablesCreated == 1)
{
//Index the chunks and their parent [fileid, n].
ReqlFunction1 chunkIx = row => { return(R.Array(row[Chunk.FilesIdJsonName], row[Chunk.NumJsonName])); };
await CreateIndex(this.chunkTableName, this.chunkIndexName, chunkIx, cancelToken)
.ConfigureAwait(false);
}
this.Mounted = true;
}
/// <summary>
/// Gets all 'completed' file revisions for a given file. Deleted and incomplete files are excluded.
/// </summary>
public static async Task <Cursor <FileInfo> > GetAllRevisionsAsync(this Bucket bucket, string filename, CancellationToken cancelToken = default)
{
filename = filename.SafePath();
var index = new { index = bucket.fileIndex };
var cursor = await bucket.fileTable
.Between(R.Array(Status.Completed, filename, R.Minval()), R.Array(Status.Completed, filename, R.Maxval()))[index]
.RunCursorAsync <FileInfo>(bucket.conn, cancelToken)
.ConfigureAwait(false);
return(cursor);
}
private void EnsureIndexesExist(Connection c)
{
const string name_client_user_index = "client_name_user";
List<string> indexes = documentTable.IndexList().Run<List<string>>(c);
if (!indexes.Contains(name_client_user_index))
{
logger.LogInformation("Creating indexes...");
RethinkDb.Driver.Ast.ReqlFunction1 pathIx = row => { return R.Array(row["client"], row["name"], row["user"]); };
documentTable.IndexCreate(name_client_user_index, pathIx).Run(c);
documentTable.IndexWait(name_client_user_index).Run(c);
logger.LogInformation("Created all indexes");
}
}
/// <summary>
/// Enumerate the file system entries of a given particular status.
/// </summary>
public static async Task <Cursor <FileInfo> > EnumerateFileEntriesAsync(Bucket bucket, string filename, Status status,
CancellationToken cancelToken = default(CancellationToken))
{
filename = filename.SafePath();
var index = new { index = bucket.fileIndex };
var cursor = await bucket.fileTable
.Between(R.Array(status, filename, R.Minval()), R.Array(status, filename, R.Maxval()))[index]
.RunCursorAsync <FileInfo>(bucket.conn, cancelToken)
.ConfigureAwait(false);
return(cursor);
}
public static int Main(string[] args)
{
const string rethinkHost = "localhost";
const string redisHost = "localhost";
const string dbName = "doc_stack_db";
const string tableName = "documents";
//indexes
const string name_client_user_index = "name_client_user";
var ip = GetIp(rethinkHost);
var c = R.Connection()
.Hostname(ip)
.Port(RethinkDBConstants.DefaultPort)
.Timeout(60)
.Connect();
List <string> dbs = R.DbList().Run <List <string> >(c);
if (!dbs.Contains(dbName))
{
R.DbCreate(dbName).Run(c);
}
var db = R.Db(dbName);
List <string> tables = db.TableList().Run <List <string> >(c);
if (!tables.Contains(tableName))
{
db.TableCreate(tableName).Run(c);
}
var documentTable = db.Table(tableName);
List <string> indexes = documentTable.IndexList().Run(c);
if (!indexes.Contains(name_client_user_index))
{
RethinkDb.Driver.Ast.ReqlFunction1 pathIx = row => { return(R.Array(row["name"], row["client"], row["user"])); };
documentTable.IndexCreate("name_client_user", pathIx).Run(c);
documentTable.IndexWait("name_client_user").Run(c);
}
try
{
var redis = OpenRedisConnection(redisHost).GetDatabase();
var definition = new { id = "", name = "", size = 0, user = "", client = "", content = "" };
while (true)
{
string json = redis.ListLeftPopAsync("documents:process:0").Result;
if (json != null)
{
var document = JsonConvert.DeserializeAnonymousType(json, definition);
Console.WriteLine($"Document '{document.name}/{document.id}/{document.user}/{document.client}' will be persisted");
var now = DateTime.UtcNow;
var toSave = new { id = document.id, user = document.user, client = document.client, name = document.name, document = document, inserted = now, updated = now, state = new[] { "persisted" } };
//first check if the document maybe already exists - right now we will just override
var x = new[] { "ocr-test2.png", "dummyClient", "dummyUser" };
//documentTable.GetAll(x).OptArg("index", "name_client_user");
//
documentTable.Insert(toSave).Run(c);
}
else
{
Thread.Sleep(500);
}
}
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.ToString());
return(1);
}
}
