public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var tbl = new Table(session, dbid, "lists", OpenTableGrbit.None))
{
JET_COLUMNID columnid;
Api.JetAddColumn(session, tbl, "created_at", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnMaybeNull,
}, null, 0, out columnid);
if (Api.TryMoveFirst(session, tbl))
{
do
{
using (var update = new Update(session, tbl, JET_prep.Replace))
{
var createdAt = Api.GetTableColumnid(session, tbl, "created_at");
Api.SetColumn(session, tbl, createdAt, SystemTime.UtcNow);
update.Save();
}
} while (Api.TryMoveNext(session, tbl));
}
SchemaCreator.CreateIndexes(session, tbl, new JET_INDEXCREATE
{
szIndexName = "by_name_and_created_at",
szKey = "+name\0+created_at\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
});
SchemaCreator.UpdateVersion(session, dbid, "5.1");
}
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var table = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, table, "by_view");
}
using (var table = new Table(session, dbid, "indexed_documents_references", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, table, "by_view");
}
using (var table = new Table(session, dbid, "reduce_keys_counts", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, table, "by_view");
}
using (var table = new Table(session, dbid, "reduce_keys_status", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, table, "by_view");
}
using (var table = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, table, "by_view");
Api.JetDeleteIndex(session, table, "by_view_and_etag");
Api.JetDeleteIndex(session, table, "by_view_bucket_and_hashed_reduce_key");
Api.JetDeleteIndex(session, table, "by_view_and_hashed_reduce_key");
SchemaCreator.CreateIndexes(session, table, new JET_INDEXCREATE
{
szIndexName = "by_view_hashed_reduce_key_and_bucket",
szKey = "+view\0+hashed_reduce_key\0+bucket\0\0",
});
}
using (var table = new Table(session, dbid, "reduce_results", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, table, "by_view");
Api.JetDeleteIndex(session, table, "by_view_level_bucket_and_hashed_reduce_key");
SchemaCreator.CreateIndexes(session, table, new JET_INDEXCREATE
{
szIndexName = "by_view_level_hashed_reduce_key_and_bucket",
szKey = "+view\0+level\0+hashed_reduce_key\0+bucket\0\0",
});
}
SchemaCreator.UpdateVersion(session, dbid, "4.5");
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var tbl = new Table(session, dbid, "tasks", OpenTableGrbit.None))
{
SchemaCreator.CreateIndexes(session, tbl, new JET_INDEXCREATE
{
szIndexName = "by_task_type",
szKey = "+task_type\0\0",
grbit = CreateIndexGrbit.IndexIgnoreNull
});
SchemaCreator.UpdateVersion(session, dbid, "5.2");
}
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var table = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
SchemaCreator.CreateIndexes(session, table, new JET_INDEXCREATE
{
szIndexName = "by_view",
szKey = "+view\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
});
}
SchemaCreator.UpdateVersion(session, dbid, "5.4");
}
public void Update(Session session, JET_DBID dbid)
{
using (var tasks = new Table(session, dbid, "tasks", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, tasks, "mergables_by_task_type");
Api.JetDeleteColumn(session, tasks, "supports_merging");
SchemaCreator.CreateIndexes(session, tasks, new JET_INDEXCREATE
{
szIndexName = "by_index_and_task_type",
szKey = "+for_index\0+task_type\0\0",
grbit = CreateIndexGrbit.IndexIgnoreNull,
});
}
SchemaCreator.UpdateVersion(session, dbid, "3.9");
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var tbl = new Table(session, dbid, "lists", OpenTableGrbit.None))
{
JET_COLUMNID columnid;
var columnids = Api.GetColumnDictionary(session, tbl);
if (columnids.ContainsKey("created_at") == false)
{
Api.JetAddColumn(session, tbl, "created_at", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnMaybeNull,
}, null, 0, out columnid);
}
int rows = 0;
if (Api.TryMoveFirst(session, tbl))
{
do
{
using (var update = new Update(session, tbl, JET_prep.Replace))
{
var createdAt = Api.GetTableColumnid(session, tbl, "created_at");
Api.SetColumn(session, tbl, createdAt, SystemTime.UtcNow);
update.Save();
}
if (rows++ % 10000 == 0)
{
output("Processed " + (rows) + " rows in lists");
Api.JetCommitTransaction(session, CommitTransactionGrbit.LazyFlush);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
} while (Api.TryMoveNext(session, tbl));
}
SchemaCreator.CreateIndexes(session, tbl, new JET_INDEXCREATE
{
szIndexName = "by_name_and_created_at",
szKey = "+name\0+created_at\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
});
SchemaCreator.UpdateVersion(session, dbid, "5.1");
}
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var tasks = new Table(session, dbid, "tasks", OpenTableGrbit.None))
{
// NOTE: the typo in the following string must stay because that's how the original index was named.
Api.JetDeleteIndex(session, tasks, "mergables_by_task_type");
Api.JetDeleteColumn(session, tasks, "supports_merging");
SchemaCreator.CreateIndexes(session, tasks, new JET_INDEXCREATE
{
szIndexName = "by_index_and_task_type",
szKey = "+for_index\0+task_type\0\0",
grbit = CreateIndexGrbit.IndexIgnoreNull,
});
}
SchemaCreator.UpdateVersion(session, dbid, "3.9");
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
Api.JetDeleteTable(session, dbid, "transactions");
Api.JetDeleteTable(session, dbid, "documents_modified_by_transaction");
using (var sr = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, sr, "by_view_level_and_hashed_reduce_key");
Api.JetDeleteIndex(session, sr, "by_view_level_bucket_and_hashed_reduce_key");
SchemaCreator.CreateIndexes(session, sr,
new JET_INDEXCREATE
{
szIndexName = "by_view_level_and_hashed_reduce_key_and_bucket",
szKey = "+view\0+level\0+hashed_reduce_key\0+bucket\0\0",
});
}
SchemaCreator.UpdateVersion(session, dbid, "4.7");
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var tbl = new Table(session, dbid, "documents", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, tbl, "by_key");
Api.JetCreateIndex2(session, tbl, new[]
{
new JET_INDEXCREATE
{
szIndexName = "by_key",
cbKey = 6,
cbKeyMost = SystemParameters.KeyMost,
cbVarSegMac = SystemParameters.KeyMost,
szKey = "+key\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull | CreateIndexGrbit.IndexUnique,
}
}, 1);
}
// So first we allocate ids and crap
// and write that to disk in a safe manner
// I might want to look at keeping a list of written files to delete if it all goes t**s up at any point
var filesToDelete = new List <string>();
var nameToIds = new Dictionary <string, int>();
var indexDefPath = Path.Combine(configuration.DataDirectory, "IndexDefinitions");
var indexDefinitions = Directory.GetFiles(indexDefPath, "*.index")
.Select(x => { filesToDelete.Add(x); return(x); })
.Select(index => JsonConvert.DeserializeObject <IndexDefinition>(File.ReadAllText(index), Default.Converters))
.ToList();
indexDefinitions.ForEach(x => x.MaxIndexOutputsPerDocument = x.MaxIndexOutputsPerDocument ?? (16 * 1024));
var transformDefinitions = Directory.GetFiles(indexDefPath, "*.transform")
.Select(x => { filesToDelete.Add(x); return(x); })
.Select(index => JsonConvert.DeserializeObject <TransformerDefinition>(File.ReadAllText(index), Default.Converters))
.ToArray();
int maxIndexId = 0;
for (var i = 0; i < indexDefinitions.Count; i++)
{
var definition = indexDefinitions[i];
definition.IndexId = i;
nameToIds[definition.Name] = definition.IndexId;
var path = Path.Combine(indexDefPath, definition.IndexId + ".index");
// TODO: This can fail, rollback
File.WriteAllText(path, JsonConvert.SerializeObject(definition, Formatting.Indented, Default.Converters));
var indexDirectory = FixupIndexName(definition.Name, configuration.IndexStoragePath);
var oldStorageDirectory = Path.Combine(configuration.IndexStoragePath, MonoHttpUtility.UrlEncode(indexDirectory));
var newStorageDirectory = Path.Combine(configuration.IndexStoragePath, definition.IndexId.ToString());
if (Directory.Exists(oldStorageDirectory)) // in-memory index
{
Directory.Move(oldStorageDirectory, newStorageDirectory);
}
maxIndexId = i;
}
for (var i = 0; i < transformDefinitions.Length; i++)
{
var definition = transformDefinitions[i];
definition.TransfomerId = maxIndexId = indexDefinitions.Count + i;
nameToIds[definition.Name] = definition.TransfomerId;
var path = Path.Combine(indexDefPath, definition.TransfomerId + ".transform");
// TODO: This can file, rollback
File.WriteAllText(path, JsonConvert.SerializeObject(definition, Formatting.Indented, Default.Converters));
}
var tablesAndColumns = new[]
{
new { table = "scheduled_reductions", column = "view" },
new { table = "mapped_results", column = "view" },
new { table = "reduce_results", column = "view" },
new { table = "reduce_keys_counts", column = "view" },
new { table = "reduce_keys_status", column = "view" },
new { table = "indexed_documents_references", column = "view" },
new { table = "tasks", column = "for_index" },
new { table = "indexes_stats", column = "key" },
new { table = "indexes_etag", column = "key" },
new { table = "indexes_stats_reduce", column = "key" }
};
foreach (var item in tablesAndColumns)
{
var newTable = item.table + "_new";
JET_TABLEID newTableId;
try
{
Api.JetDeleteTable(session, dbid, newTable);
}
catch (Exception)
{
//if there is no such table - then it is not important
//this is a precaution against partially failed upgrade process
}
using (var sr = new Table(session, dbid, item.table, OpenTableGrbit.None))
{
Api.JetCreateTable(session, dbid, newTable, 1, 80, out newTableId);
var existingColumns = Api.GetTableColumns(session, sr).ToList();
var existingIndexes = Api.GetTableIndexes(session, sr).ToList();
foreach (var column in existingColumns)
{
JET_COLUMNDEF columnDef = null;
Api.JetGetColumnInfo(session, dbid, item.table, column.Name, out columnDef);
JET_COLUMNID newColumndId;
if (column.Name == item.column)
{
Api.JetAddColumn(session, newTableId, item.column, new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, null, 0, out newColumndId);
}
else
{
var defaultValue = column.DefaultValue == null ? null : column.DefaultValue.ToArray();
var defaultValueLength = defaultValue == null ? 0 : defaultValue.Length;
{
Api.JetAddColumn(session, newTableId, column.Name, columnDef, defaultValue,
defaultValueLength, out newColumndId);
}
}
}
foreach (var index in existingIndexes)
{
var indexDesc = String.Join("\0", index.IndexSegments.Select(x => "+" + x.ColumnName)) +
"\0\0";
SchemaCreator.CreateIndexes(session, newTableId, new JET_INDEXCREATE()
{
szIndexName = index.Name,
szKey = indexDesc,
grbit = index.Grbit
});
}
var rows = 0;
using (var destTable = new Table(session, dbid, newTable, OpenTableGrbit.None))
{
Api.MoveBeforeFirst(session, sr);
Api.MoveBeforeFirst(session, destTable);
while (Api.TryMoveNext(session, sr))
{
using (var insert = new Update(session, destTable, JET_prep.Insert))
{
bool save = true;
foreach (var column in existingColumns)
{
var destcolumn = Api.GetTableColumnid(session, destTable, column.Name);
if (column.Name == item.column)
{
var viewName = Api.RetrieveColumnAsString(session, sr, column.Columnid,
Encoding.Unicode);
int value;
if (nameToIds.TryGetValue(viewName, out value) == false)
{
insert.Cancel();
save = false;
break;
}
Api.SetColumn(session, destTable, destcolumn, value);
}
else if ((column.Grbit & ColumndefGrbit.ColumnAutoincrement) == ColumndefGrbit.None)
{
var value = Api.RetrieveColumn(session, sr, column.Columnid);
Api.SetColumn(session, destTable, destcolumn, value);
}
}
if (save)
{
insert.Save();
}
}
if (rows++ % 10000 == 0)
{
output("Processed " + (rows) + " rows in " + item.table);
Api.JetCommitTransaction(session, CommitTransactionGrbit.LazyFlush);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
}
}
output("Processed " + (rows - 1) + " rows in " + item.table + ", and done with this table");
}
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
Api.JetDeleteTable(session, dbid, item.table);
Api.JetRenameTable(session, dbid, newTable, item.table);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
filesToDelete.ForEach(File.Delete);
UpdateLastIdentityForIndexes(session, dbid, maxIndexId + 1);
SchemaCreator.UpdateVersion(session, dbid, "5.0");
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
using (var tbl = new Table(session, dbid, "documents", OpenTableGrbit.None))
{
Api.JetDeleteIndex(session, tbl, "by_key");
Api.JetCreateIndex2(session, tbl, new[]
{
new JET_INDEXCREATE
{
szIndexName = "by_key",
cbKey = 6,
cbKeyMost = SystemParameters.KeyMost,
cbVarSegMac = SystemParameters.KeyMost,
szKey = "+key\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull | CreateIndexGrbit.IndexUnique,
}
}, 1);
}
// So first we allocate ids and crap
// and write that to disk in a safe manner
// I might want to look at keeping a list of written files to delete if it all goes t**s up at any point
var filesToDelete = new List <string>();
var nameToIds = new Dictionary <string, int>();
var indexDefPath = Path.Combine(configuration.DataDirectory, "IndexDefinitions");
var indexDefinitions = Directory.GetFiles(indexDefPath, "*.index")
.Select(x => { filesToDelete.Add(x); return(x); })
.Select(index => JsonConvert.DeserializeObject <IndexDefinition>(File.ReadAllText(index), Default.Converters))
.ToArray();
var transformDefinitions = Directory.GetFiles(indexDefPath, "*.transform")
.Select(x => { filesToDelete.Add(x); return(x); })
.Select(index => JsonConvert.DeserializeObject <TransformerDefinition>(File.ReadAllText(index), Default.Converters))
.ToArray();
int maxIndexId = 0;
for (var i = 0; i < indexDefinitions.Length; i++)
{
var definition = indexDefinitions[i];
definition.IndexId = i;
nameToIds[definition.Name] = definition.IndexId;
var path = Path.Combine(indexDefPath, definition.IndexId + ".index");
// TODO: This can fail, rollback
File.WriteAllText(path, JsonConvert.SerializeObject(definition, Formatting.Indented, Default.Converters));
var indexDirectory = FixupIndexName(definition.Name, configuration.IndexStoragePath);
var oldStorageDirectory = Path.Combine(configuration.IndexStoragePath, MonoHttpUtility.UrlEncode(indexDirectory));
var newStorageDirectory = Path.Combine(configuration.IndexStoragePath, definition.IndexId.ToString());
// TODO: This can fail, rollback
Directory.Move(oldStorageDirectory, newStorageDirectory);
maxIndexId = i;
}
for (var i = 0; i < transformDefinitions.Length; i++)
{
var definition = transformDefinitions[i];
definition.TransfomerId = maxIndexId = indexDefinitions.Length + i;
nameToIds[definition.Name] = definition.TransfomerId;
var path = Path.Combine(indexDefPath, definition.TransfomerId + ".transform");
// TODO: This can file, rollback
File.WriteAllText(path, JsonConvert.SerializeObject(definition, Formatting.Indented, Default.Converters));
}
var tablesAndColumns = new[]
{
new { table = "scheduled_reductions", column = "view" },
new { table = "mapped_results", column = "view" },
new { table = "reduce_results", column = "view" },
new { table = "reduce_keys_counts", column = "view" },
new { table = "reduce_keys_status", column = "view" },
new { table = "indexed_documents_references", column = "view" },
new { table = "tasks", column = "for_index" },
new { table = "indexes_stats", column = "key" },
new { table = "indexes_etag", column = "key" },
new { table = "indexes_stats_reduce", column = "key" }
};
foreach (var item in tablesAndColumns)
{
// If it's the primary key we need to be a bit clever
// cos we can't just rename this and copy data over
// cos indexes
if (item.column == "key")
{
var newTable = item.table + "_new";
JET_TABLEID newTableId;
using (var sr = new Table(session, dbid, item.table, OpenTableGrbit.None))
{
Api.JetCreateTable(session, dbid, newTable, 1, 80, out newTableId);
var existingColumns = Api.GetTableColumns(session, sr);
var existingIndexes = Api.GetTableIndexes(session, sr);
foreach (var column in existingColumns)
{
JET_COLUMNDEF columnDef = null;
Api.JetGetColumnInfo(session, dbid, item.table, column.Name, out columnDef);
JET_COLUMNID newColumndId;
if (column.Name == item.column)
{
Api.JetAddColumn(session, newTableId, item.column, new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, null, 0, out newColumndId);
}
else
{
var defaultValue = column.DefaultValue == null ? null : column.DefaultValue.ToArray();
var defaultValueLength = defaultValue == null ? 0 : defaultValue.Length;
{
Api.JetAddColumn(session, newTableId, column.Name, columnDef, defaultValue,
defaultValueLength, out newColumndId);
}
}
}
foreach (var index in existingIndexes)
{
var indexDesc = String.Join("\0", index.IndexSegments.Select(x => "+" + x.ColumnName)) +
"\0\0";
SchemaCreator.CreateIndexes(session, newTableId, new JET_INDEXCREATE()
{
szIndexName = index.Name,
szKey = indexDesc,
grbit = index.Grbit
});
}
var rows = 0;
using (var destTable = new Table(session, dbid, newTable, OpenTableGrbit.None))
{
Api.MoveBeforeFirst(session, sr);
Api.MoveBeforeFirst(session, destTable);
while (Api.TryMoveNext(session, sr))
{
using (var insert = new Update(session, destTable, JET_prep.Insert))
{
foreach (var column in existingColumns)
{
var destcolumn = Api.GetTableColumnid(session, destTable, column.Name);
if (column.Name == item.column)
{
var viewName = Api.RetrieveColumnAsString(session, sr, column.Columnid,
Encoding.Unicode);
Api.SetColumn(session, destTable, destcolumn, nameToIds[viewName]);
}
else
{
var value = Api.RetrieveColumn(session, sr, column.Columnid);
Api.SetColumn(session, destTable, destcolumn, value);
}
}
insert.Save();
}
if (rows++ % 10000 == 0)
{
output("Processed " + (rows - 1) + " rows in " + item.table);
continue;
}
}
}
}
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
Api.JetDeleteTable(session, dbid, item.table);
Api.JetRenameTable(session, dbid, newTable, item.table);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
else
{
// We can do an in-place update
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
using (var sr = new Table(session, dbid, item.table, OpenTableGrbit.None))
{
Api.JetRenameColumn(session, sr, item.column, item.column + "_old", RenameColumnGrbit.None);
JET_COLUMNID columnid;
Api.JetAddColumn(session, sr, item.column, new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
var rows = 0;
Api.MoveBeforeFirst(session, sr);
while (Api.TryMoveNext(session, sr))
{
var viewNameId = Api.GetTableColumnid(session, sr, item.column + "_old");
var viewIdId = Api.GetTableColumnid(session, sr, item.column);
using (var update = new Update(session, sr, JET_prep.Replace))
{
var viewName = Api.RetrieveColumnAsString(session, sr, viewNameId, Encoding.Unicode);
Api.SetColumn(session, sr, viewIdId, nameToIds[viewName]);
update.Save();
}
if (rows++ % 10000 == 0)
{
output("Processed " + (rows - 1) + " rows in " + item.table);
continue;
}
}
foreach (var index in Api.GetTableIndexes(session, sr))
{
if (index.IndexSegments.Any(x => x.ColumnName == item.column + "_old"))
{
Api.JetDeleteIndex(session, sr, index.Name);
var indexKeys = string.Join("", index.IndexSegments.Select(x => string.Format("+{0}\0", x.ColumnName))) + "\0";
indexKeys = indexKeys.Replace("_old", "");
SchemaCreator.CreateIndexes(session, sr, new JET_INDEXCREATE
{
szIndexName = index.Name,
szKey = indexKeys
});
}
}
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
Api.JetDeleteColumn(session, sr, item.column + "_old");
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
}
UpdateLastIdentityForIndexes(session, dbid, maxIndexId + 1);
}
filesToDelete.ForEach(File.Delete);
SchemaCreator.UpdateVersion(session, dbid, "4.8");
}
private void CreateReduceKeysCountsTable(Session session, JET_DBID dbid)
{
JET_TABLEID tableid;
Api.JetCreateTable(session, dbid, "reduce_keys_counts", 1, 80, out tableid);
JET_COLUMNID columnid;
try
{
Api.JetAddColumn(session, tableid, "id", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnAutoincrement | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "view", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp() | ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var defaultValue = BitConverter.GetBytes(0);
Api.JetAddColumn(session, tableid, "mapped_items_count", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL | ColumndefGrbit.ColumnEscrowUpdate
}, defaultValue, defaultValue.Length, out columnid);
SchemaCreator.CreateIndexes(session, tableid,
new JET_INDEXCREATE
{
szIndexName = "by_id",
szKey = "+id\0\0",
grbit = CreateIndexGrbit.IndexPrimary
},
new JET_INDEXCREATE
{
szIndexName = "by_view",
szKey = "+view\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_view_and_hashed_reduce_key",
szKey = "+view\0+hashed_reduce_key\0+reduce_key\0\0",
grbit = CreateIndexGrbit.IndexUnique
});
}
finally
{
Api.JetCloseTable(session, tableid);
}
}
private void CreateReduceKeysStatusTable(Session session, JET_DBID dbid)
{
JET_TABLEID tableid;
Api.JetCreateTable(session, dbid, "reduce_keys_status", 1, 80, out tableid);
JET_COLUMNID columnid;
Api.JetAddColumn(session, tableid, "id", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnAutoincrement | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "view", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp() | ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "reduce_type", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
SchemaCreator.CreateIndexes(session, tableid,
new JET_INDEXCREATE
{
szIndexName = "by_id",
szKey = "+id\0\0",
grbit = CreateIndexGrbit.IndexPrimary
},
new JET_INDEXCREATE
{
szIndexName = "by_view",
szKey = "+view\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_view_and_hashed_reduce_key",
szKey = "+view\0+hashed_reduce_key\0+reduce_key\0\0",
grbit = CreateIndexGrbit.IndexUnique
});
}
private void CreateIndexedDocumentsReferencesTable(Session session, JET_DBID dbid)
{
JET_TABLEID tableid;
Api.JetCreateTable(session, dbid, "indexed_documents_references", 1, 80, out tableid);
JET_COLUMNID columnid;
Api.JetAddColumn(session, tableid, "id", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnAutoincrement | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "view", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "key", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "ref", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
SchemaCreator.CreateIndexes(session, tableid,
new JET_INDEXCREATE
{
szIndexName = "by_id",
szKey = "+id\0\0",
grbit = CreateIndexGrbit.IndexPrimary
},
new JET_INDEXCREATE
{
szIndexName = "by_key",
szKey = "+key\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_view",
szKey = "+key\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_view_and_key",
szKey = "+view\0+key\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_ref",
szKey = "+ref\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
});
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
var i = 0;
JET_COLUMNID columnid;
using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
{
Api.JetAddColumn(session, scheduledReductions, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var scheduledReductionsColumns = Api.GetColumnDictionary(session, scheduledReductions);
Api.MoveBeforeFirst(session, scheduledReductions);
while (Api.TryMoveNext(session, scheduledReductions))
{
using (var update = new Update(session, scheduledReductions, JET_prep.Replace))
{
var reduceKey = Api.RetrieveColumnAsString(session, scheduledReductions, scheduledReductionsColumns["reduce_key"]);
Api.SetColumn(session, scheduledReductions, scheduledReductionsColumns["hashed_reduce_key"],
DocumentStorageActions.HashReduceKey(reduceKey));
update.Save();
}
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in scheduled_reductions");
}
}
Api.JetDeleteIndex(session, scheduledReductions, "by_view_level_reduce_key_and_bucket");
SchemaCreator.CreateIndexes(session, scheduledReductions, new JET_INDEXCREATE
{
szIndexName = "by_view_level_bucket_and_hashed_reduce_key",
szKey = "+view\0+level\0+bucket\0+hashed_reduce_key\0\0",
});
}
output("Finished processing scheduled_reductions");
i = 0;
using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
Api.JetAddColumn(session, mappedResults, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var mappedResultsColumns = Api.GetColumnDictionary(session, mappedResults);
Api.MoveBeforeFirst(session, mappedResults);
while (Api.TryMoveNext(session, mappedResults))
{
using (var update = new Update(session, mappedResults, JET_prep.Replace))
{
var reduceKey = Api.RetrieveColumnAsString(session, mappedResults, mappedResultsColumns["reduce_key"]);
Api.SetColumn(session, mappedResults, mappedResultsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey));
update.Save();
}
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in mapped_results");
}
}
Api.JetDeleteIndex(session, mappedResults, "by_view_reduce_key_and_bucket");
SchemaCreator.CreateIndexes(session, mappedResults, new JET_INDEXCREATE
{
szIndexName = "by_view_bucket_and_hashed_reduce_key",
szKey = "+view\0+bucket\0+hashed_reduce_key\0\0",
});
}
output("Finished processing mapped_results");
i = 0;
using (var reduceResults = new Table(session, dbid, "reduce_results", OpenTableGrbit.None))
{
Api.JetAddColumn(session, reduceResults, "hashed_reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 20,
grbit = ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
var reduceResultsColumns = Api.GetColumnDictionary(session, reduceResults);
Api.MoveBeforeFirst(session, reduceResults);
while (Api.TryMoveNext(session, reduceResults))
{
using (var update = new Update(session, reduceResults, JET_prep.Replace))
{
var reduceKey = Api.RetrieveColumnAsString(session, reduceResults, reduceResultsColumns["reduce_key"]);
Api.SetColumn(session, reduceResults, reduceResultsColumns["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(reduceKey));
update.Save();
}
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in reduce_results");
}
}
Api.JetDeleteIndex(session, reduceResults, "by_view_level_reduce_key_and_bucket");
Api.JetDeleteIndex(session, reduceResults, "by_view_level_reduce_key_and_source_bucket");
SchemaCreator.CreateIndexes(session, reduceResults,
new JET_INDEXCREATE
{
szIndexName = "by_view_level_bucket_and_hashed_reduce_key",
szKey = "+view\0+level\0+bucket\0+hashed_reduce_key\0\0",
},
new JET_INDEXCREATE
{
szIndexName = "by_view_level_source_bucket_and_hashed_reduce_key",
szKey = "+view\0+level\0+source_bucket\0+hashed_reduce_key\0\0",
});
}
output("Finished processing reduce_results");
SchemaCreator.UpdateVersion(session, dbid, "4.0");
}
public void CreateListsTable(Session session, JET_DBID dbid)
{
JET_TABLEID tableid;
Api.JetCreateTable(session, dbid, "lists", 1, 80, out tableid);
JET_COLUMNID columnid;
Api.JetAddColumn(session, tableid, "id", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnAutoincrement | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "name", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "key", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "data", new JET_COLUMNDEF
{
coltyp = JET_coltyp.LongBinary,
grbit = ColumndefGrbit.ColumnTagged
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "etag", new JET_COLUMNDEF
{
cbMax = 16,
coltyp = JET_coltyp.Binary,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL,
}, null, 0, out columnid);
SchemaCreator.CreateIndexes(session, tableid,
new JET_INDEXCREATE
{
szIndexName = "by_id",
szKey = "+id\0\0",
grbit = CreateIndexGrbit.IndexPrimary
},
new JET_INDEXCREATE
{
szIndexName = "by_name_and_etag",
szKey = "+name\0+etag\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_name_and_key",
szKey = "+name\0+key\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull | CreateIndexGrbit.IndexUnique
});
}
public void Update(Session session, JET_DBID dbid, Action <string> output)
{
var i = 0;
CreateReduceKeysCountsTable(session, dbid);
CreateReduceKeysStatusTable(session, dbid);
var countsPerKeyPerIndex = new Dictionary <string, Dictionary <string, int> >(StringComparer.OrdinalIgnoreCase);
using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
SchemaCreator.CreateIndexes(session, mappedResults, new JET_INDEXCREATE
{
szIndexName = "by_view_and_hashed_reduce_key",
szKey = "+view\0+hashed_reduce_key\0\0",
});
var columnDictionary = Api.GetColumnDictionary(session, mappedResults);
Api.MoveBeforeFirst(session, mappedResults);
while (Api.TryMoveNext(session, mappedResults))
{
var index = Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["view"], Encoding.Unicode);
var reduceKey = Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["reduce_key"], Encoding.Unicode);
var countPerKey = countsPerKeyPerIndex.GetOrAdd(index);
countPerKey[reduceKey] = countPerKey.GetOrDefault(reduceKey) + 1;
if (i++ % 10000 == 0)
{
output("Processed " + (i - 1) + " rows in mapped_results");
}
}
}
output("Finished processing mapped_results");
using (var reduceKeys = new Table(session, dbid, "reduce_keys_status", OpenTableGrbit.None))
{
var columnDictionary = Api.GetColumnDictionary(session, reduceKeys);
foreach (var countPerKey in countsPerKeyPerIndex)
{
foreach (var keyCount in countPerKey.Value)
{
using (var update = new Update(session, reduceKeys, JET_prep.Insert))
{
Api.SetColumn(session, reduceKeys, columnDictionary["view"], countPerKey.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["reduce_key"], keyCount.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(keyCount.Key));
Api.SetColumn(session, reduceKeys, columnDictionary["reduce_type"], (int)ReduceType.MultiStep);
update.Save();
}
}
}
}
output("Finished processing reduce_keys_status");
using (var reduceKeys = new Table(session, dbid, "reduce_keys_counts", OpenTableGrbit.None))
{
var columnDictionary = Api.GetColumnDictionary(session, reduceKeys);
foreach (var countPerKey in countsPerKeyPerIndex)
{
foreach (var keyCount in countPerKey.Value)
{
using (var update = new Update(session, reduceKeys, JET_prep.Insert))
{
Api.SetColumn(session, reduceKeys, columnDictionary["view"], countPerKey.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["reduce_key"], keyCount.Key, Encoding.Unicode);
Api.SetColumn(session, reduceKeys, columnDictionary["hashed_reduce_key"], DocumentStorageActions.HashReduceKey(keyCount.Key));
Api.SetColumn(session, reduceKeys, columnDictionary["mapped_items_count"], keyCount.Value);
update.Save();
}
}
}
}
output("Finished processing reduce_keys_counts");
using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
{
SchemaCreator.CreateIndexes(session, scheduledReductions, new JET_INDEXCREATE
{
szIndexName = "by_view_level_and_hashed_reduce_key",
szKey = "+view\0+level\0+hashed_reduce_key\0\0",
});
}
output("Finished processing scheduled_reductions");
SchemaCreator.UpdateVersion(session, dbid, "4.1");
}
private void CreateScheduledReductionsTable(Session session, JET_DBID dbid)
{
JET_TABLEID tableid;
Api.JetCreateTable(session, dbid, "scheduled_reductions", 1, 80, out tableid);
JET_COLUMNID columnid;
Api.JetAddColumn(session, tableid, "id", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnAutoincrement | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "view", new JET_COLUMNDEF
{
cbMax = 2048,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "reduce_key", new JET_COLUMNDEF
{
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = SchemaCreator.ColumnNotNullIfOnHigherThanWindowsXp()
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "etag", new JET_COLUMNDEF
{
cbMax = 16,
coltyp = JET_coltyp.Binary,
grbit = ColumndefGrbit.ColumnNotNULL | ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "timestamp", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnNotNULL | ColumndefGrbit.ColumnFixed
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "bucket", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
Api.JetAddColumn(session, tableid, "level", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
SchemaCreator.CreateIndexes(session, tableid,
new JET_INDEXCREATE
{
szIndexName = "by_id",
szKey = "+id\0\0",
grbit = CreateIndexGrbit.IndexPrimary
},
new JET_INDEXCREATE
{
szIndexName = "by_view",
szKey = "+view\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
},
new JET_INDEXCREATE
{
szIndexName = "by_view_level_reduce_key_and_bucket",
szKey = "+view\0+level\0+reduce_key\0+bucket\0\0",
});
}
