/// <summary> /// Mounts the bucket. Mount is necessary before using a bucket to ensure the existence of tables and indexes. /// </summary> public async Task MountAsync(CancellationToken cancelToken = default) { if (this.Mounted) { return; } var filesTableResult = await EnsureTable(this.fileTableName, cancelToken) .ConfigureAwait(false); if (filesTableResult.TablesCreated == 1) { //index the file paths of completed files and status ReqlFunction1 pathIx = row => { return(R.Array(row[FileInfo.StatusJsonName], row[FileInfo.FileNameJsonName], row[FileInfo.FinishedDateJsonName])); }; await CreateIndex(this.fileTableName, this.fileIndex, pathIx, cancelToken) .ConfigureAwait(false); //prefix IX ReqlFunction1 prefixIx = doc => { //return r.array(doc[FileInfo.FileNameJsonName].split("/").slice(1, -1), doc[FileInfo.FinishedDateJsonName]); return(R.Branch(doc[FileInfo.StatusJsonName].Eq(Status.Completed), R.Array(doc[FileInfo.FileNameJsonName].Split("/").Slice(1, -1), doc[FileInfo.FinishedDateJsonName]), R.Error())); }; await CreateIndex(this.fileTableName, this.fileIndexPrefix, prefixIx, cancelToken) .ConfigureAwait(false); } // CHUNK TAABLE INDEXES var chunkTableResult = await EnsureTable(this.chunkTableName, cancelToken) .ConfigureAwait(false); if (chunkTableResult.TablesCreated == 1) { //Index the chunks and their parent [fileid, n]. ReqlFunction1 chunkIx = row => { return(R.Array(row[Chunk.FilesIdJsonName], row[Chunk.NumJsonName])); }; await CreateIndex(this.chunkTableName, this.chunkIndexName, chunkIx, cancelToken) .ConfigureAwait(false); } this.Mounted = true; }
/// <summary> /// Gets all 'completed' file revisions for a given file. Deleted and incomplete files are excluded. /// </summary> public static async Task <Cursor <FileInfo> > GetAllRevisionsAsync(this Bucket bucket, string filename, CancellationToken cancelToken = default) { filename = filename.SafePath(); var index = new { index = bucket.fileIndex }; var cursor = await bucket.fileTable .Between(R.Array(Status.Completed, filename, R.Minval()), R.Array(Status.Completed, filename, R.Maxval()))[index] .RunCursorAsync <FileInfo>(bucket.conn, cancelToken) .ConfigureAwait(false); return(cursor); }
private void EnsureIndexesExist(Connection c) { const string name_client_user_index = "client_name_user"; List<string> indexes = documentTable.IndexList().Run<List<string>>(c); if (!indexes.Contains(name_client_user_index)) { logger.LogInformation("Creating indexes..."); RethinkDb.Driver.Ast.ReqlFunction1 pathIx = row => { return R.Array(row["client"], row["name"], row["user"]); }; documentTable.IndexCreate(name_client_user_index, pathIx).Run(c); documentTable.IndexWait(name_client_user_index).Run(c); logger.LogInformation("Created all indexes"); } }
/// <summary> /// Enumerate the file system entries of a given particular status. /// </summary> public static async Task <Cursor <FileInfo> > EnumerateFileEntriesAsync(Bucket bucket, string filename, Status status, CancellationToken cancelToken = default(CancellationToken)) { filename = filename.SafePath(); var index = new { index = bucket.fileIndex }; var cursor = await bucket.fileTable .Between(R.Array(status, filename, R.Minval()), R.Array(status, filename, R.Maxval()))[index] .RunCursorAsync <FileInfo>(bucket.conn, cancelToken) .ConfigureAwait(false); return(cursor); }
public static int Main(string[] args) { const string rethinkHost = "localhost"; const string redisHost = "localhost"; const string dbName = "doc_stack_db"; const string tableName = "documents"; //indexes const string name_client_user_index = "name_client_user"; var ip = GetIp(rethinkHost); var c = R.Connection() .Hostname(ip) .Port(RethinkDBConstants.DefaultPort) .Timeout(60) .Connect(); List <string> dbs = R.DbList().Run <List <string> >(c); if (!dbs.Contains(dbName)) { R.DbCreate(dbName).Run(c); } var db = R.Db(dbName); List <string> tables = db.TableList().Run <List <string> >(c); if (!tables.Contains(tableName)) { db.TableCreate(tableName).Run(c); } var documentTable = db.Table(tableName); List <string> indexes = documentTable.IndexList().Run(c); if (!indexes.Contains(name_client_user_index)) { RethinkDb.Driver.Ast.ReqlFunction1 pathIx = row => { return(R.Array(row["name"], row["client"], row["user"])); }; documentTable.IndexCreate("name_client_user", pathIx).Run(c); documentTable.IndexWait("name_client_user").Run(c); } try { var redis = OpenRedisConnection(redisHost).GetDatabase(); var definition = new { id = "", name = "", size = 0, user = "", client = "", content = "" }; while (true) { string json = redis.ListLeftPopAsync("documents:process:0").Result; if (json != null) { var document = JsonConvert.DeserializeAnonymousType(json, definition); Console.WriteLine($"Document '{document.name}/{document.id}/{document.user}/{document.client}' will be persisted"); var now = DateTime.UtcNow; var toSave = new { id = document.id, user = document.user, client = document.client, name = document.name, document = document, inserted = now, updated = now, state = new[] { "persisted" } }; //first check if the document maybe already exists - right now we will just override var x = new[] { "ocr-test2.png", "dummyClient", "dummyUser" }; //documentTable.GetAll(x).OptArg("index", "name_client_user"); // documentTable.Insert(toSave).Run(c); } else { Thread.Sleep(500); } } } catch (Exception ex) { Console.Error.WriteLine(ex.ToString()); return(1); } }