public void Update(Session session, JET_DBID dbid, Action <string> output) { var i = 0; CreateReduceKeysCountsTable(session, dbid); CreateReduceKeysStatusTable(session, dbid); var countsPerKeyPerIndex = new Dictionary <string, Dictionary <string, int> >(StringComparer.OrdinalIgnoreCase); using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None)) { SchemaCreator.CreateIndexes(session, mappedResults, new JET_INDEXCREATE { szIndexName = "by_view_and_hashed_reduce_key", szKey = "+view\0+hashed_reduce_key\0\0", }); var columnDictionary = Api.GetColumnDictionary(session, mappedResults); Api.MoveBeforeFirst(session, mappedResults); while (Api.TryMoveNext(session, mappedResults)) { var index = Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["view"], Encoding.Unicode); var reduceKey = Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["reduce_key"], Encoding.Unicode); var countPerKey = countsPerKeyPerIndex.GetOrAdd(index); countPerKey[reduceKey] = countPerKey.GetOrDefault(reduceKey) + 1; if (i++ % 10000 == 0) { output("Processed " + (i - 1) + " rows in mapped_results"); } } } output("Finished processing mapped_results"); using (var reduceKeys = new Table(session, dbid, "reduce_keys_status", OpenTableGrbit.None)) { var columnDictionary = Api.GetColumnDictionary(session, reduceKeys); foreach (var countPerKey in countsPerKeyPerIndex) { foreach (var keyCount in countPerKey.Value) { using (var update = new Update(session, reduceKeys, JET_prep.Insert)) { Api.SetColumn(session, reduceKeys, columnDictionary["view"], countPerKey.Key, Encoding.Unicode); Api.SetColumn(session, reduceKeys, columnDictionary["reduce_key"], keyCount.Key, Encoding.Unicode); Api.SetColumn(session, reduceKeys, columnDictionary["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(keyCount.Key)); Api.SetColumn(session, reduceKeys, columnDictionary["reduce_type"], (int)ReduceType.MultiStep); update.Save(); } } } } output("Finished processing reduce_keys_status"); using (var reduceKeys = new Table(session, dbid, "reduce_keys_counts", OpenTableGrbit.None)) { var columnDictionary = Api.GetColumnDictionary(session, reduceKeys); foreach (var countPerKey in countsPerKeyPerIndex) { foreach (var keyCount in countPerKey.Value) { using (var update = new Update(session, reduceKeys, JET_prep.Insert)) { Api.SetColumn(session, reduceKeys, columnDictionary["view"], countPerKey.Key, Encoding.Unicode); Api.SetColumn(session, reduceKeys, columnDictionary["reduce_key"], keyCount.Key, Encoding.Unicode); Api.SetColumn(session, reduceKeys, columnDictionary["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(keyCount.Key)); Api.SetColumn(session, reduceKeys, columnDictionary["mapped_items_count"], keyCount.Value); update.Save(); } } } } output("Finished processing reduce_keys_counts"); using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None)) { SchemaCreator.CreateIndexes(session, scheduledReductions, new JET_INDEXCREATE { szIndexName = "by_view_level_and_hashed_reduce_key", szKey = "+view\0+level\0+hashed_reduce_key\0\0", }); } output("Finished processing scheduled_reductions"); SchemaCreator.UpdateVersion(session, dbid, "4.1"); }
public void Update(Session session, JET_DBID dbid, Action <string> output) { var i = 0; JET_COLUMNID columnid; using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None)) { Api.JetAddColumn(session, scheduledReductions, "hashed_reduce_key", new JET_COLUMNDEF { coltyp = JET_coltyp.Binary, cbMax = 20, grbit = ColumndefGrbit.ColumnFixed }, null, 0, out columnid); var scheduledReductionsColumns = Api.GetColumnDictionary(session, scheduledReductions); Api.MoveBeforeFirst(session, scheduledReductions); while (Api.TryMoveNext(session, scheduledReductions)) { using (var update = new Update(session, scheduledReductions, JET_prep.Replace)) { var reduceKey = Api.RetrieveColumnAsString(session, scheduledReductions, scheduledReductionsColumns["reduce_key"]); Api.SetColumn(session, scheduledReductions, scheduledReductionsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey)); update.Save(); } if (i++ % 10000 == 0) { output("Processed " + (i - 1) + " rows in scheduled_reductions"); } } Api.JetDeleteIndex(session, scheduledReductions, "by_view_level_reduce_key_and_bucket"); SchemaCreator.CreateIndexes(session, scheduledReductions, new JET_INDEXCREATE { szIndexName = "by_view_level_bucket_and_hashed_reduce_key", szKey = "+view\0+level\0+bucket\0+hashed_reduce_key\0\0", }); } output("Finished processing scheduled_reductions"); i = 0; using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None)) { Api.JetAddColumn(session, mappedResults, "hashed_reduce_key", new JET_COLUMNDEF { coltyp = JET_coltyp.Binary, cbMax = 20, grbit = ColumndefGrbit.ColumnFixed }, null, 0, out columnid); var mappedResultsColumns = Api.GetColumnDictionary(session, mappedResults); Api.MoveBeforeFirst(session, mappedResults); while (Api.TryMoveNext(session, mappedResults)) { using (var update = new Update(session, mappedResults, JET_prep.Replace)) { var reduceKey = Api.RetrieveColumnAsString(session, mappedResults, mappedResultsColumns["reduce_key"]); Api.SetColumn(session, mappedResults, mappedResultsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey)); update.Save(); } if (i++ % 10000 == 0) { output("Processed " + (i - 1) + " rows in mapped_results"); } } Api.JetDeleteIndex(session, mappedResults, "by_view_reduce_key_and_bucket"); SchemaCreator.CreateIndexes(session, mappedResults, new JET_INDEXCREATE { szIndexName = "by_view_bucket_and_hashed_reduce_key", szKey = "+view\0+bucket\0+hashed_reduce_key\0\0", }); } output("Finished processing mapped_results"); i = 0; using (var reduceResults = new Table(session, dbid, "reduce_results", OpenTableGrbit.None)) { Api.JetAddColumn(session, reduceResults, "hashed_reduce_key", new JET_COLUMNDEF { coltyp = JET_coltyp.Binary, cbMax = 20, grbit = ColumndefGrbit.ColumnFixed }, null, 0, out columnid); var reduceResultsColumns = Api.GetColumnDictionary(session, reduceResults); Api.MoveBeforeFirst(session, reduceResults); while (Api.TryMoveNext(session, reduceResults)) { using (var update = new Update(session, reduceResults, JET_prep.Replace)) { var reduceKey = Api.RetrieveColumnAsString(session, reduceResults, reduceResultsColumns["reduce_key"]); Api.SetColumn(session, reduceResults, reduceResultsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey)); update.Save(); } if (i++ % 10000 == 0) { output("Processed " + (i - 1) + " rows in reduce_results"); } } Api.JetDeleteIndex(session, reduceResults, "by_view_level_reduce_key_and_bucket"); Api.JetDeleteIndex(session, reduceResults, "by_view_level_reduce_key_and_source_bucket"); SchemaCreator.CreateIndexes(session, reduceResults, new JET_INDEXCREATE { szIndexName = "by_view_level_bucket_and_hashed_reduce_key", szKey = "+view\0+level\0+bucket\0+hashed_reduce_key\0\0", }, new JET_INDEXCREATE { szIndexName = "by_view_level_source_bucket_and_hashed_reduce_key", szKey = "+view\0+level\0+source_bucket\0+hashed_reduce_key\0\0", }); } output("Finished processing reduce_results"); SchemaCreator.UpdateVersion(session, dbid, "4.0"); }