public void Update(Session session, JET_DBID dbid, Action<string> output)
{
using (var tbl = new Table(session, dbid, "tasks", OpenTableGrbit.None))
{
int rows = 0;
if (Api.TryMoveFirst(session, tbl))
{
var taskTypeColumnId = Api.GetTableColumnid(session, tbl, "task_type");
do
{
using (var update = new Update(session, tbl, JET_prep.Replace))
{
var taskType = Api.RetrieveColumnAsString(session, tbl, taskTypeColumnId, Encoding.Unicode);
Api.SetColumn(session, tbl, taskTypeColumnId, taskType, Encoding.ASCII);
update.Save();
}
if (rows++ % 10000 == 0)
{
output("Processed " + (rows) + " rows in tasks");
Api.JetCommitTransaction(session, CommitTransactionGrbit.LazyFlush);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
} while (Api.TryMoveNext(session, tbl));
}
SchemaCreator.UpdateVersion(session, dbid, "5.3");
}
}
public static void DumpTable(Session session, Table table, Stream stream)
{
using (var writer = new StreamWriter(stream))
{
var cols = Api.GetTableColumns(session, table).ToArray();
foreach (var col in cols)
{
writer.Write(col);
writer.Write(",");
}
writer.WriteLine("#");
Api.MoveBeforeFirst(session, table);
int count = 0;
while (Api.TryMoveNext(session, table))
{
foreach (var col in cols)
{
var val = GetvalueFromTable(session, table, col) ?? "NULL";
writer.Write(val);
writer.Write(",");
}
writer.WriteLine(++count);
}
writer.Flush();
}
}
public void Update(Session session, JET_DBID dbid, Action<string> output)
{
using (var tbl = new Table(session, dbid, "lists", OpenTableGrbit.None))
{
JET_COLUMNID columnid;
Api.JetAddColumn(session, tbl, "created_at", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnMaybeNull,
}, null, 0, out columnid);
if (Api.TryMoveFirst(session, tbl))
{
do
{
using (var update = new Update(session, tbl, JET_prep.Replace))
{
var createdAt = Api.GetTableColumnid(session, tbl, "created_at");
Api.SetColumn(session, tbl, createdAt, SystemTime.UtcNow);
update.Save();
}
} while (Api.TryMoveNext(session, tbl));
}
SchemaCreator.CreateIndexes(session, tbl, new JET_INDEXCREATE
{
szIndexName = "by_name_and_created_at",
szKey = "+name\0+created_at\0\0",
grbit = CreateIndexGrbit.IndexDisallowNull
});
SchemaCreator.UpdateVersion(session, dbid, "5.1");
}
}
public void Update(Session session, JET_DBID dbid)
{
using (var tx = new Transaction(session))
{
using (var files = new Table(session, dbid, "files", OpenTableGrbit.None))
{
const string indexDef = "+etag\0\0";
Api.JetCreateIndex(session, files, "by_etag", CreateIndexGrbit.IndexDisallowNull, indexDef, indexDef.Length,
100);
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using (var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "2.7", Encoding.Unicode);
update.Save();
}
}
}
tx.Commit(CommitTransactionGrbit.None);
}
}
private static object GetvalueFromTable(Session session, Table table, ColumnInfo col)
{
switch (col.Coltyp)
{
case JET_coltyp.Long:
return Api.RetrieveColumnAsInt32(session, table, col.Columnid);
case JET_coltyp.DateTime:
return Api.RetrieveColumnAsDateTime(session, table, col.Columnid);
case JET_coltyp.Binary:
var bytes = Api.RetrieveColumn(session, table, col.Columnid);
if (bytes == null)
return null;
if (bytes.Length == 16)
return new Guid(bytes);
return Convert.ToBase64String(bytes);
case JET_coltyp.LongText:
case JET_coltyp.Text:
var str = Api.RetrieveColumnAsString(session, table, col.Columnid);
if (str == null)
return null;
if (str.Contains("\""))
return "\"" + str.Replace("\"", "\"\"") + "\"";
return str;
case JET_coltyp.LongBinary:
return "long binary val";
default:
throw new ArgumentOutOfRangeException("don't know how to handle coltype: " + col.Coltyp);
}
}
public void Update(Session session, JET_DBID dbid, Action<string> output)
{
using (var table = new Table(session, dbid, "indexes_stats", OpenTableGrbit.None))
{
byte[] defaultValue = BitConverter.GetBytes(1);
JET_COLUMNID columnid;
Api.JetAddColumn(session, table, "priority", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Long,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, defaultValue, defaultValue.Length, out columnid);
defaultValue = BitConverter.GetBytes(0);
Api.JetAddColumn(session, table, "created_timestamp", new JET_COLUMNDEF
{
cbMax = 8, //64 bits
coltyp = JET_coltyp.Binary,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, defaultValue, defaultValue.Length, out columnid);
Api.JetAddColumn(session, table, "last_indexing_time", new JET_COLUMNDEF
{
cbMax = 8, //64 bits
coltyp = JET_coltyp.Binary,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, defaultValue, defaultValue.Length, out columnid);
}
SchemaCreator.UpdateVersion(session, dbid, "4.6");
}
public void Update(Session session, JET_DBID dbid)
{
using (var tx = new Transaction(session))
{
using (var tasks = new Table(session, dbid, "tasks", OpenTableGrbit.None))
{
JET_COLUMNID columnid;
Api.JetAddColumn(session, tasks, "added_at",new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using (var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "2.5", Encoding.Unicode);
update.Save();
}
}
}
tx.Commit(CommitTransactionGrbit.None);
}
}
protected override CrawlerQueueEntry PopImpl()
{
return m_EsentInstance.Cursor((session, dbid) =>
{
using (Transaction transaction = new Transaction(session))
{
using (Table table = new Table(session, dbid, EsentTableDefinitions.QueueTableName, OpenTableGrbit.None))
{
if (Api.TryMoveFirst(session, table))
{
byte[] data = Api.RetrieveColumn(session, table, dataColumn.columnid);
Api.JetDelete(session, table);
using (Table table2 = new Table(session, dbid, EsentTableDefinitions.GlobalsTableName, OpenTableGrbit.None))
{
Api.EscrowUpdate(session, table2, queueCountColumn.columnid, -1);
}
transaction.Commit(CommitTransactionGrbit.None);
return data.FromBinary<CrawlerQueueEntry>();
}
}
transaction.Rollback();
return null;
}
});
}
protected void GetColumnIds(
JET_SESID sessionId, Table table, out JET_COLUMNID projectColumnId, out JET_COLUMNID projectNameColumnId, out JET_COLUMNID documentColumnId)
{
projectColumnId = Api.GetTableColumnid(sessionId, table, ProjectColumnName);
projectNameColumnId = Api.GetTableColumnid(sessionId, table, ProjectNameColumnName);
documentColumnId = Api.GetTableColumnid(sessionId, table, DocumentColumnName);
}
protected AbstractActions(JET_INSTANCE instance, ColumnsInformation columnsInformation, string database, Guid instanceId)
{
try
{
this.instanceId = instanceId;
ColumnsInformation = columnsInformation;
session = new Session(instance);
transaction = new Transaction(session);
Api.JetOpenDatabase(session, database, null, out dbid, OpenDatabaseGrbit.None);
queues = new Table(session, dbid, "queues", OpenTableGrbit.None);
subqueues = new Table(session, dbid, "subqueues", OpenTableGrbit.None);
txs = new Table(session, dbid, "transactions", OpenTableGrbit.None);
recovery = new Table(session, dbid, "recovery", OpenTableGrbit.None);
outgoing = new Table(session, dbid, "outgoing", OpenTableGrbit.None);
outgoingHistory = new Table(session, dbid, "outgoing_history", OpenTableGrbit.None);
recveivedMsgs = new Table(session, dbid, "recveived_msgs", OpenTableGrbit.None);
}
catch (Exception)
{
Dispose();
throw;
}
}
public void InitColumDictionaries(JET_INSTANCE instance, string database)
{
using (var session = new Session(instance))
{
var dbid = JET_DBID.Nil;
try
{
Api.JetOpenDatabase(session, database, null, out dbid, OpenDatabaseGrbit.None);
using (var usage = new Table(session, dbid, "usage", OpenTableGrbit.None))
UsageColumns = Api.GetColumnDictionary(session, usage);
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
DetailsColumns = Api.GetColumnDictionary(session, details);
using (var pages = new Table(session, dbid, "pages", OpenTableGrbit.None))
PagesColumns = Api.GetColumnDictionary(session, pages);
using (var files = new Table(session, dbid, "files", OpenTableGrbit.None))
FilesColumns = Api.GetColumnDictionary(session, files);
using (var signatures = new Table(session, dbid, "signatures", OpenTableGrbit.None))
SignaturesColumns = Api.GetColumnDictionary(session, signatures);
using (var config = new Table(session, dbid, "config", OpenTableGrbit.None))
ConfigColumns = Api.GetColumnDictionary(session, config);
}
finally
{
if (Equals(dbid, JET_DBID.Nil) == false)
Api.JetCloseDatabase(session, dbid, CloseDatabaseGrbit.None);
}
}
}
public void Update(Session session, JET_DBID dbid)
{
using(var tx = new Transaction(session))
{
using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
{
JET_COLUMNID columnid;
Api.JetAddColumn(session, mappedResults, "reduce_key_and_view_hashed", new JET_COLUMNDEF
{
cbMax = 32,
coltyp = JET_coltyp.Binary,
grbit = ColumndefGrbit.ColumnNotNULL
}, null, 0, out columnid);
const string indexDef = "+reduce_key_and_view_hashed\0\0";
Api.JetCreateIndex(session, mappedResults, "by_reduce_key_and_view_hashed",
CreateIndexGrbit.IndexDisallowNull, indexDef, indexDef.Length,
100);
Api.JetDeleteIndex(session, mappedResults, "by_view_and_reduce_key");
var columnDictionary = Api.GetColumnDictionary(session, mappedResults);
Api.MoveBeforeFirst(session, mappedResults);
while (Api.TryMoveNext(session, mappedResults))
{
using (var update = new Update(session, mappedResults, JET_prep.Replace))
{
var computeHash = MapReduceIndex.ComputeHash(
Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["view"],
Encoding.Unicode),
Api.RetrieveColumnAsString(session, mappedResults, columnDictionary["reduce_key"],
Encoding.Unicode));
Api.SetColumn(session, mappedResults, columnDictionary["reduce_key_and_view_hashed"],
computeHash);
update.Save();
}
}
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using(var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "2.2", Encoding.Unicode);
update.Save();
}
}
}
tx.Commit(CommitTransactionGrbit.None);
}
}
public void Update(Session session, JET_DBID dbid, Action<string> output)
{
using (var table = new Table(session, dbid, "config", OpenTableGrbit.DenyRead | OpenTableGrbit.PermitDDL))
{
JET_COLUMNID newMetadataColumnId;
Api.JetAddColumn(session, table, "metadata_new", new JET_COLUMNDEF
{
cbMax = 1024*512,
coltyp = JET_coltyp.LongText,
cp = JET_CP.Unicode,
grbit = ColumndefGrbit.ColumnNotNULL
}, null, 0, out newMetadataColumnId);
}
using (var table = new Table(session, dbid, "config", OpenTableGrbit.None))
{
Api.MoveBeforeFirst(session, table);
var rows = 0;
var columnDictionary = Api.GetColumnDictionary(session, table);
var metadataColumn = columnDictionary["metadata"];
var nameColumn = columnDictionary["name"];
var newMetadataColumn = columnDictionary["metadata_new"];
while (Api.TryMoveNext(session, table))
{
using (var insert = new Update(session, table, JET_prep.Replace))
{
var name = Api.RetrieveColumnAsString(session, table, nameColumn, Encoding.Unicode);
var metadata = Api.RetrieveColumnAsString(session, table, metadataColumn, Encoding.Unicode);
var fixedMetadata = GuidToEtagMigrationInConfigurations(metadata, name);
Api.SetColumn(session, table, newMetadataColumn, fixedMetadata, Encoding.Unicode);
insert.Save();
}
if (rows++ % 100 == 0)
{
output("Processed " + (rows) + " rows from metadata column in config table");
Api.JetCommitTransaction(session, CommitTransactionGrbit.LazyFlush);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
}
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
// they cannot be run in transaction scope
Api.JetDeleteColumn(session, table, "metadata");
Api.JetRenameColumn(session, table, "metadata_new", "metadata", RenameColumnGrbit.None);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
SchemaCreator.UpdateVersion(session, dbid, "0.5");
}
public override void Initialize(JET_SESID sessionId, JET_DBID databaseId)
{
using (var table = new Table(sessionId, databaseId, TableName, OpenTableGrbit.ReadOnly))
{
_idColumnId = Api.GetTableColumnid(sessionId, table, IdColumnName);
_identifierColumnId = Api.GetTableColumnid(sessionId, table, IdentifierColumnName);
}
}
public void Update(Session session, JET_DBID dbid)
{
Transaction tx;
using (tx = new Transaction(session))
{
using ( var tbl = new Table(session, dbid, "indexes_stats",
OpenTableGrbit.PermitDDL | OpenTableGrbit.DenyRead | OpenTableGrbit.DenyWrite))
{
var columnDictionary = Api.GetColumnDictionary(session, tbl);
if (columnDictionary.ContainsKey("reduce_attempts"))
Api.JetDeleteColumn(session, tbl, "reduce_attempts");
if (columnDictionary.ContainsKey("reduce_errors"))
Api.JetDeleteColumn(session, tbl, "reduce_errors");
if (columnDictionary.ContainsKey("reduce_successes"))
Api.JetDeleteColumn(session, tbl, "reduce_successes");
}
CreateIndexesStatsReduce(session, dbid);
using (var stats = new Table(session, dbid, "indexes_stats",OpenTableGrbit.None))
using (var reduce = new Table(session, dbid, "indexes_stats_reduce", OpenTableGrbit.None))
{
var tblKeyColumn = Api.GetColumnDictionary(session, stats)["key"];
var reduceKeyCol = Api.GetColumnDictionary(session, reduce)["key"];
Api.MoveBeforeFirst(session, stats);
while (Api.TryMoveNext(session, stats))
{
using(var update = new Update(session, reduce, JET_prep.Insert))
{
var indexName = Api.RetrieveColumnAsString(session, stats, tblKeyColumn, Encoding.Unicode);
Api.SetColumn(session, reduce, reduceKeyCol, indexName, Encoding.Unicode);
update.Save();
}
}
}
tx.Commit(CommitTransactionGrbit.LazyFlush);
tx.Dispose();
tx = new Transaction(session);
}
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using (var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "3.4", Encoding.Unicode);
update.Save();
}
}
tx.Commit(CommitTransactionGrbit.None);
tx.Dispose();
}
private Table OpenSchema(JET_SESID sesid, JET_DBID dbid, out JET_COLUMNID primaryColumnId, out JET_COLUMNID secondaryColumnId)
{
var table = new Table(sesid, dbid, "table", OpenTableGrbit.None);
primaryColumnId = Api.GetTableColumnid(sesid, table, "key");
secondaryColumnId = Api.GetTableColumnid(sesid, table, "data");
return table;
}
public override void Initialize(JET_SESID sessionId, JET_DBID databaseId)
{
using (var table = new Table(sessionId, databaseId, TableName, OpenTableGrbit.ReadOnly))
{
GetColumnIds(sessionId, table, out _projectColumnId, out _projectNameColumnId);
_nameColumnId = Api.GetTableColumnid(sessionId, table, NameColumnName);
_valueColumnId = Api.GetTableColumnid(sessionId, table, ValueColumnName);
}
}
public void Update(Session session, JET_DBID dbid)
{
Transaction tx;
using (tx = new Transaction(session))
{
var count = 0;
const int rowsInTxCount = 100;
var tablesWithEtags = new[]
{
new {Table = "indexes_stats", Column = "last_indexed_etag"},
new {Table = "documents", Column = "etag"},
new {Table = "mapped_results", Column = "etag"},
new {Table = "files", Column = "etag"},
new {Table = "documents_modified_by_transaction", Column = "etag"},
};
foreach (var tablesWithEtag in tablesWithEtags)
{
using (var tbl = new Table(session, dbid, tablesWithEtag.Table, OpenTableGrbit.None))
{
var columnid = Api.GetColumnDictionary(session, tbl)[tablesWithEtag.Column];
Api.MoveBeforeFirst(session, tbl);
while (Api.TryMoveNext(session, tbl))
{
using (var update = new Update(session, tbl, JET_prep.Replace))
{
Api.SetColumn(session, tbl, columnid, Guid.Empty.TransformToValueForEsentSorting());
update.Save();
}
if (count++%rowsInTxCount == 0)
{
tx.Commit(CommitTransactionGrbit.LazyFlush);
tx.Dispose();
tx = new Transaction(session);
}
}
tx.Commit(CommitTransactionGrbit.LazyFlush);
tx.Dispose();
tx = new Transaction(session);
}
}
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using (var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "3.2", Encoding.Unicode);
update.Save();
}
}
tx.Commit(CommitTransactionGrbit.None);
tx.Dispose();
}
}
/// <summary>
/// See base
/// </summary>
public override void Insert(string tableName, IEnumerable <TableRow> rows)
{
if (tableName == null)
{
throw new ArgumentNullException(nameof(tableName));
}
if (rows == null)
{
throw new ArgumentNullException(nameof(rows));
}
var rowsList = rows.ToList();
if (rowsList.Count == 0)
{
return;
}
if (!TableRow.AreDistinct(rowsList))
{
throw new StorageException(ErrorCode.DuplicateKey, null);
}
using (ETable table = OpenTable(tableName, true))
{
JET_TABLEID tableId = table.JetTableid;
Dictionary <string, JET_COLUMNID> columns = EnsureColumnsExist(tableName, tableId, rowsList);
using (var transaction = new Transaction(_jetSession))
{
foreach (TableRow row in rowsList)
{
using (var update = new Update(_jetSession, table.JetTableid, JET_prep.Insert))
{
SetValue(tableId, columns, row.Id);
foreach (KeyValuePair <string, DynamicValue> column in row)
{
SetValue(tableId, columns[column.Key], column.Value);
}
try
{
update.Save();
}
catch (EsentKeyDuplicateException ex)
{
throw new StorageException(ErrorCode.DuplicateKey, ex);
}
}
}
transaction.Commit(CommitTransactionGrbit.None);
}
}
}
public override void Initialize(JET_SESID sessionId, JET_DBID databaseId)
{
using (var table = new Table(sessionId, databaseId, TableName, OpenTableGrbit.ReadOnly))
{
GetColumnIds(sessionId, table, out _projectColumnId, out _projectNameColumnId, out _documentColumnId);
_identifierColumnId = Api.GetTableColumnid(sessionId, table, IdentifierColumnName);
_locationsColumnId = Api.GetTableColumnid(sessionId, table, LocationsColumnName);
}
}
public override void AddColumn(ColumnDefinition columnDef)
{
using (var transaction = new Transaction(connection.session))
using (var table = new Microsoft.Isam.Esent.Interop.Table(connection.session, connection.dbid, name, OpenTableGrbit.None)) {
var tableCreator = new EseTableCreator(connection);
tableCreator.AddColumn(table, columnDef);
transaction.Commit(CommitTransactionGrbit.None);
}
RefreshTableInfo();
}
public void InitColumDictionaries(JET_INSTANCE instance, string database)
{
using (var session = new Session(instance))
{
var dbid = JET_DBID.Nil;
try
{
Api.JetOpenDatabase(session, database, null, out dbid, OpenDatabaseGrbit.None);
using (var documents = new Table(session, dbid, "documents", OpenTableGrbit.None))
DocumentsColumns = Api.GetColumnDictionary(session, documents);
using (var lists = new Table(session, dbid, "lists", OpenTableGrbit.None))
ListsColumns = Api.GetColumnDictionary(session, lists);
using (var tasks = new Table(session, dbid, "tasks", OpenTableGrbit.None))
TasksColumns = Api.GetColumnDictionary(session, tasks);
using (var files = new Table(session, dbid, "files", OpenTableGrbit.None))
FilesColumns = Api.GetColumnDictionary(session, files);
using (var scheduledReductions = new Table(session, dbid, "scheduled_reductions", OpenTableGrbit.None))
ScheduledReductionColumns = Api.GetColumnDictionary(session, scheduledReductions);
using (var indexStats = new Table(session, dbid, "indexes_stats", OpenTableGrbit.None))
IndexesStatsColumns = Api.GetColumnDictionary(session, indexStats);
using (var indexStatsReduce = new Table(session, dbid, "indexes_stats_reduce", OpenTableGrbit.None))
IndexesStatsReduceColumns = Api.GetColumnDictionary(session, indexStatsReduce);
using (var indexEtags = new Table(session, dbid, "indexes_etag", OpenTableGrbit.None))
IndexesEtagsColumns = Api.GetColumnDictionary(session, indexEtags);
using (var mappedResults = new Table(session, dbid, "mapped_results", OpenTableGrbit.None))
MappedResultsColumns = Api.GetColumnDictionary(session, mappedResults);
using (var reduceResults = new Table(session, dbid, "reduce_results", OpenTableGrbit.None))
ReduceResultsColumns = Api.GetColumnDictionary(session, reduceResults);
using (var indexed_documents_references = new Table(session, dbid, "indexed_documents_references", OpenTableGrbit.None))
IndexedDocumentsReferencesColumns = Api.GetColumnDictionary(session, tableid: indexed_documents_references);
using (
var documentsModifiedByTransactions = new Table(session, dbid, "documents_modified_by_transaction",
OpenTableGrbit.None))
DocumentsModifiedByTransactionsColumns = Api.GetColumnDictionary(session,
documentsModifiedByTransactions);
using (var transactions = new Table(session, dbid, "transactions", OpenTableGrbit.None))
TransactionsColumns = Api.GetColumnDictionary(session, transactions);
using (var identity = new Table(session, dbid, "identity_table", OpenTableGrbit.None))
IdentityColumns = Api.GetColumnDictionary(session, identity);
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
DetailsColumns = Api.GetColumnDictionary(session, details);
using (var queue = new Table(session, dbid, "queue", OpenTableGrbit.None))
QueueColumns = Api.GetColumnDictionary(session, queue);
using (var reduceKeys = new Table(session, dbid, "reduce_keys_counts", OpenTableGrbit.None))
ReduceKeysCountsColumns = Api.GetColumnDictionary(session, reduceKeys);
using (var reduceKeys = new Table(session, dbid, "reduce_keys_status", OpenTableGrbit.None))
ReduceKeysStatusColumns = Api.GetColumnDictionary(session, reduceKeys);
}
finally
{
if (Equals(dbid, JET_DBID.Nil) == false)
Api.JetCloseDatabase(session, dbid, CloseDatabaseGrbit.None);
}
}
}
private ETable OpenTable(string tableName, bool createIfNotExists)
{
ETable result;
try
{
result = new ETable(_jetSession, _jetDbId, tableName, OpenTableGrbit.None);
}
catch (EsentObjectNotFoundException)
{
result = null;
}
if (result == null && createIfNotExists)
{
//create table
using (var transaction = new Transaction(_jetSession))
{
Api.JetCreateTable(_jetSession, _jetDbId, tableName, 16, 100, out JET_TABLEID tableId);
try
{
//create index columns
var columnDef = new JET_COLUMNDEF
{
coltyp = JET_coltyp.Text,
cp = JET_CP.Unicode
};
//add key columns
Api.JetAddColumn(_jetSession, tableId, "PartitionKey", columnDef, null, 0, out JET_COLUMNID columnId);
Api.JetAddColumn(_jetSession, tableId, "RowKey", columnDef, null, 0, out columnId);
//index key columns
//+ indicates ascending order, - descending
string indexDefPrimary = $"+{PartitionKeyName}\0+{RowKeyName}\0\0";
Api.JetCreateIndex(_jetSession, tableId, PrimaryIndexName, CreateIndexGrbit.IndexPrimary, indexDefPrimary, indexDefPrimary.Length, 100);
string indexDefPartition = $"+{PartitionKeyName}\0\0";
Api.JetCreateIndex(_jetSession, tableId, PartitionIndexName, CreateIndexGrbit.None, indexDefPartition, indexDefPartition.Length, 100);
transaction.Commit(CommitTransactionGrbit.LazyFlush);
}
finally
{
Api.JetCloseTable(_jetSession, tableId);
}
result = new ETable(_jetSession, _jetDbId, tableName, OpenTableGrbit.None);
}
}
return(result);
}
public QueueActions(Session session, JET_DBID dbid, string queueName, string[] subqueues, AbstractActions actions, Action<int> changeNumberOfMessages)
{
_queueName = queueName;
_subqueues = subqueues;
_actions = actions;
_changeNumberOfMessages = changeNumberOfMessages;
var msgs = new Table(session, dbid, queueName, OpenTableGrbit.None);
var msgsHistory = new Table(session, dbid, queueName + "_history", OpenTableGrbit.None);
_messages = new EsentTable(session, msgs);
_messageHistory = new EsentTable(session, msgsHistory);
}
public void Update(Session session, JET_DBID dbid)
{
using (var tx = new Transaction(session))
{
using (var indexStats = new Table(session, dbid, "indexes_stats", OpenTableGrbit.None))
using (var documents = new Table(session, dbid, "documents", OpenTableGrbit.None))
using (var documentsInTx = new Table(session, dbid, "documents_modified_by_transaction", OpenTableGrbit.None))
{
var defaultValue = Guid.Empty.ToByteArray();
JET_COLUMNID columnid;
Api.JetAddColumn(session, indexStats, "last_indexed_etag", new JET_COLUMNDEF
{
coltyp = JET_coltyp.Binary,
cbMax = 16,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, defaultValue, defaultValue.Length, out columnid);
defaultValue = BitConverter.GetBytes(SystemTime.UtcNow.AddSeconds(-1).ToOADate());
Api.JetAddColumn(session, indexStats, "last_indexed_timestamp", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL
}, defaultValue, defaultValue.Length, out columnid);
Api.JetAddColumn(session, documents, "last_modified", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL,
}, defaultValue, defaultValue.Length, out columnid);
Api.JetAddColumn(session, documentsInTx, "last_modified", new JET_COLUMNDEF
{
coltyp = JET_coltyp.DateTime,
grbit = ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL,
}, defaultValue, defaultValue.Length, out columnid);
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using (var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "2.6", Encoding.Unicode);
update.Save();
}
}
}
tx.Commit(CommitTransactionGrbit.None);
}
}
public QueueActions(Session session, JET_DBID dbid, string queueName, string[] subqueues, AbstractActions actions, Action<int> changeNumberOfMessages)
{
this.session = session;
this.queueName = queueName;
this.subqueues = subqueues;
this.actions = actions;
this.changeNumberOfMessages = changeNumberOfMessages;
msgs = new Table(session, dbid, queueName, OpenTableGrbit.None);
msgsColumns = Api.GetColumnDictionary(session, msgs);
msgsHistory = new Table(session, dbid, queueName + "_history", OpenTableGrbit.None);
msgsHistoryColumns = Api.GetColumnDictionary(session, msgsHistory);
}
public void Update(Session session, JET_DBID dbid)
{
// those actions are no longer required, and are actually removed on the next version
// also, they don't work on non empty tables, so we skip them
//using (tx = new Transaction(session))
//{
// using (var tbl = new Table(session, dbid, "indexes_stats", OpenTableGrbit.PermitDDL | OpenTableGrbit.DenyRead|OpenTableGrbit.DenyWrite))
// {
// JET_COLUMNID columnid;
// var defaultValue = BitConverter.GetBytes(0);
// Api.JetAddColumn(session, tbl, "reduce_attempts", new JET_COLUMNDEF
// {
// coltyp = JET_coltyp.Long,
// grbit =
// ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnEscrowUpdate | ColumndefGrbit.ColumnNotNULL
// }, defaultValue, defaultValue.Length, out columnid);
// Api.JetAddColumn(session, tbl, "reduce_errors", new JET_COLUMNDEF
// {
// coltyp = JET_coltyp.Long,
// grbit =
// ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnEscrowUpdate | ColumndefGrbit.ColumnNotNULL
// }, defaultValue, defaultValue.Length, out columnid);
// Api.JetAddColumn(session, tbl, "reduce_successes", new JET_COLUMNDEF
// {
// coltyp = JET_coltyp.Long,
// grbit =
// ColumndefGrbit.ColumnFixed | ColumndefGrbit.ColumnNotNULL | ColumndefGrbit.ColumnEscrowUpdate
// }, defaultValue, defaultValue.Length, out columnid);
// }
// tx.Commit(CommitTransactionGrbit.LazyFlush);
// tx.Dispose();
// tx = new Transaction(session);
//}
using (var tx = new Transaction(session))
{
using (var details = new Table(session, dbid, "details", OpenTableGrbit.None))
{
Api.JetMove(session, details, JET_Move.First, MoveGrbit.None);
var columnids = Api.GetColumnDictionary(session, details);
using (var update = new Update(session, details, JET_prep.Replace))
{
Api.SetColumn(session, details, columnids["schema_version"], "3.3", Encoding.Unicode);
update.Save();
}
}
tx.Commit(CommitTransactionGrbit.None);
}
}
protected void EnsureTableForUpdating()
{
if (_table == null)
{
_table = new Table(_session.SessionId, _session.DatabaseId, _tableName, OpenTableGrbit.Updatable);
}
if (_transaction == null)
{
_transaction = new Transaction(_session.SessionId);
}
}
private Session OpenSession(JET_INSTANCE instance, out Table table, out JET_COLUMNID primaryColumnId, out JET_COLUMNID secondaryColumnId)
{
var session = new Session(instance);
Api.JetAttachDatabase2(session, _database, 0, AttachDatabaseGrbit.None);
JET_DBID dbid;
Api.JetOpenDatabase(session, _database, null, out dbid, OpenDatabaseGrbit.None);
table = OpenSchema(session, dbid, out primaryColumnId, out secondaryColumnId);
return session;
}
public void Update(Session session, JET_DBID dbid, Action<string> output)
{
using (var table = new Table(session, dbid, "pages", OpenTableGrbit.DenyRead | OpenTableGrbit.PermitDDL))
{
JET_COLUMNID newDataColumnId;
Api.JetAddColumn(session, table, "data_new", new JET_COLUMNDEF
{
cbMax = 4 * StorageConstants.MaxPageSize, // handle possible data expansion because of codecs usage
coltyp = JET_coltyp.LongBinary,
grbit = ColumndefGrbit.ColumnMaybeNull
}, null, 0, out newDataColumnId);
}
using (var table = new Table(session, dbid, "pages", OpenTableGrbit.None))
{
Api.MoveBeforeFirst(session, table);
var dataColumnId = Api.GetTableColumnid(session, table, "data");
var newDataColumnId = Api.GetTableColumnid(session, table, "data_new");
var rows = 0;
while (Api.TryMoveNext(session, table))
{
using (var insert = new Update(session, table, JET_prep.Replace))
{
var value = Api.RetrieveColumn(session, table, dataColumnId);
Api.SetColumn(session, table, newDataColumnId, value);
insert.Save();
}
if (rows++ % 1000 == 0)
{
output("Processed " + (rows) + " rows from data column in pages table");
Api.JetCommitTransaction(session, CommitTransactionGrbit.LazyFlush);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
}
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
// they cannot be run in transaction scope
Api.JetDeleteColumn(session, table, "data");
Api.JetRenameColumn(session, table, "data_new", "data", RenameColumnGrbit.None);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
SchemaCreator.UpdateVersion(session, dbid, "0.4");
}
public void IinsertAndReadOneMessage()
{
TestHelper.CreateDatabase("EsentTestsDummy");
using (var instance = new Instance("MyInstanceName"))
{
instance.Parameters.CircularLog = true;
instance.Init();
using (var session = new Session(instance))
{
JET_DBID dbid;
Api.JetAttachDatabase(session, DatabaseName, AttachDatabaseGrbit.None);
Api.JetOpenDatabase(session, DatabaseName, null, out dbid, OpenDatabaseGrbit.None);
JET_TABLEID tableid;
if (Api.TryOpenTable(session, dbid, "EsentTestsDummymessages", OpenTableGrbit.None, out tableid))
{
IDictionary<string, JET_COLUMNID> columnids = Api.GetColumnDictionary(session, tableid);
JET_COLUMNID columnidMessage = columnids["message"];
JET_COLUMNID columnidMetaData = columnids["metadata"];
Assert.IsInstanceOfType(columnidMessage, typeof(JET_COLUMNID));
Assert.IsInstanceOfType(columnidMetaData, typeof(JET_COLUMNID));
AddMessage(session, tableid, ref columnidMessage, ref columnidMetaData);
var results = TestHelper.DumpByIndex(session, tableid, null, columnids);
Assert.IsNotNull(results);
Assert.IsTrue(results.Count == 1);
Assert.AreEqual("Hi this is the message", results[0]);
}
else
{
using (var table = new Table(session, dbid, "EsentTestsDummymessages", OpenTableGrbit.None))
{
IDictionary<string, JET_COLUMNID> columnids = Api.GetColumnDictionary(session, table);
JET_COLUMNID columnidMessage = columnids["message"];
JET_COLUMNID columnidMetaData = columnids["metadata"];
Assert.IsInstanceOfType(columnidMessage, typeof(JET_COLUMNID));
Assert.IsInstanceOfType(columnidMetaData, typeof(JET_COLUMNID));
AddMessage(session, table, ref columnidMessage, ref columnidMetaData);
var results = TestHelper.DumpByIndex(session, table, null, columnids);
Assert.IsNotNull(results);
Assert.IsTrue(results.Count == 1);
Assert.AreEqual("Hi this is the message", results[0]);
}
}
}
}
}
public void Update(Session session, JET_DBID dbid, Action<string> output)
{
var tableAndColumns = new[]
{
new {Table = "indexes_stats", Column = "last_indexed_timestamp"},
new {Table = "indexes_stats_reduce", Column = "last_reduced_timestamp"},
new {Table = "transactions", Column = "timeout"},
new {Table = "documents", Column = "last_modified"},
new {Table = "documents_modified_by_transaction", Column = "last_modified"},
new {Table = "scheduled_reductions", Column = "timestamp"},
new {Table = "scheduled_reductions", Column = "timestamp"},
new {Table = "mapped_results", Column = "timestamp"},
new {Table = "reduce_results", Column = "timestamp"},
new {Table = "tasks", Column = "added_at"},
};
int rows = 0;
foreach (var tableAndColumn in tableAndColumns)
{
using (var table = new Table(session, dbid, tableAndColumn.Table, OpenTableGrbit.None))
{
Api.MoveBeforeFirst(session, table);
while (Api.TryMoveNext(session, table))
{
var columnid = Api.GetTableColumnid(session, table, tableAndColumn.Column);
using (var update = new Update(session, table, JET_prep.Replace))
{
var bytes = Api.RetrieveColumn(session, table, columnid);
var date = DateTime.FromOADate(BitConverter.ToDouble(bytes, 0));
Api.SetColumn(session, table, columnid, date.ToBinary());
update.Save();
}
if (rows++ % 10000 == 0)
{
output("Processed " + (rows - 1) + " rows in " + tableAndColumn.Table);
continue;
}
// pulsing transaction
Api.JetCommitTransaction(session, CommitTransactionGrbit.None);
Api.JetBeginTransaction2(session, BeginTransactionGrbit.None);
}
}
output("Finished processing " + tableAndColumn.Table);
}
SchemaCreator.UpdateVersion(session, dbid, "4.3");
}
private IEnumerable <TableRow> InternalGet(string tableName, string partitionKey, string rowKey, int maxRecords)
{
using (ETable table = OpenTable(tableName, false))
{
if (table == null)
{
return(Enumerable.Empty <TableRow>());
}
Dictionary <string, JET_COLUMNID> columns = _tableNameToColumnNameToId[tableName];
if (!SeekToPkRk(table.JetTableid, partitionKey, rowKey))
{
return(Enumerable.Empty <TableRow>());
}
return(ReadAllRows(tableName, table.JetTableid, columns, rowKey == null ? maxRecords : 1));
}
}
/// <summary>
/// See base
/// </summary>
public override void Delete(string tableName, IEnumerable <TableRowId> rowIds)
{
if (rowIds == null)
{
return;
}
using (ETable table = OpenTable(tableName, false))
{
foreach (TableRowId id in rowIds)
{
if (SeekToPkRk(table.JetTableid, id.PartitionKey, id.RowKey))
{
Api.JetDelete(_jetSession, table.JetTableid); //deletes current record
}
}
}
}
/// <summary>
///
/// </summary>
/// <param name="session"></param>
/// <param name="dbid"></param>
private static void ExportLinkTable(Session session, JET_DBID dbid)
{
using (System.IO.StreamWriter file = new System.IO.StreamWriter("linktable.csv"))
using (var table = new Microsoft.Isam.Esent.Interop.Table(session, dbid, "link_table", OpenTableGrbit.ReadOnly))
{
// Extract and cache the columns from the "link_table" table
List <ColumnInfo> columns = new List <ColumnInfo>();
foreach (ColumnInfo column in Api.GetTableColumns(session, dbid, "link_table"))
{
columns.Add(column);
}
// Write out the column headers
int index = 0;
foreach (ColumnInfo column in columns)
{
index += 1;
file.Write(column.Name);
if (index != columns.Count())
{
file.Write("\t");
}
}
file.WriteLine("");
Api.JetSetTableSequential(session, table, SetTableSequentialGrbit.None);
Api.MoveBeforeFirst(session, table);
List <dynamic> temp = new List <dynamic>();
int currentRow = 0;
string formattedData = "";
while (Api.TryMoveNext(session, table))
{
currentRow++;
dynamic data = new ExpandoObject();
IDictionary <string, object> obj = data;
foreach (ColumnInfo column in columns)
{
formattedData = GetFormattedColumnData(session, table, column);
obj.Add(column.Name, formattedData.Replace("\0", string.Empty));
}
// Now write out each columns data
index = 0;
foreach (ColumnInfo column in columns)
{
index += 1;
file.Write(obj[column.Name]);
if (index != columns.Count())
{
file.Write("\t");
}
;
}
file.WriteLine("");
}
Api.JetResetTableSequential(session, table, ResetTableSequentialGrbit.None);
}
}
/// <summary>
///
/// </summary>
/// <param name="session"></param>
/// <param name="dbid"></param>
private static void ExportDataTable(Session session, JET_DBID dbid)
{
// Extract and cache the columns from the "datatable" table. Note
// that we are only interested in the columns needed for "ntdsextract"
List <ColumnInfo> columns = new List <ColumnInfo>();
foreach (ColumnInfo column in Api.GetTableColumns(session, dbid, "datatable"))
{
if (userColumns.Contains(column.Name) == false)
{
continue;
}
columns.Add(column);
}
using (System.IO.StreamWriter file = new System.IO.StreamWriter("datatable.csv"))
using (var table = new Microsoft.Isam.Esent.Interop.Table(session, dbid, "datatable", OpenTableGrbit.ReadOnly))
{
// Write out the column headers
int index = 0;
foreach (string property in userColumns)
{
index += 1;
file.Write(property);
if (index != userColumns.Count())
{
file.Write("\t");
}
}
file.WriteLine("");
Api.JetSetTableSequential(session, table, SetTableSequentialGrbit.None);
Api.MoveBeforeFirst(session, table);
int currentRow = 0;
string formattedData = "";
while (Api.TryMoveNext(session, table))
{
currentRow++;
dynamic data = new ExpandoObject();
IDictionary <string, object> obj = data;
foreach (ColumnInfo column in columns)
{
formattedData = GetFormattedColumnData(session, table, column);
// The first row has a null "PDNT_col" value which causes issues with the "ntdsxtract" scripts.
// esedbexport seems to have some other value, esedbexport parses the data, rather than using the API
if (column.Name == "PDNT_col")
{
if (formattedData.Length == 0)
{
obj.Add(column.Name, "0");
continue;
}
}
//add the column
//replace null terminators with empty strings
//replace newlines with a single space
//trim any whitespace from the value
obj.Add(column.Name, formattedData.Replace("\0", string.Empty).Replace("\n", " ").Trim());
}
// Now write out each columns data
index = 0;
foreach (string property in userColumns)
{
index += 1;
file.Write(obj[property]);
if (index != userColumns.Count())
{
file.Write("\t");
}
}
file.WriteLine("");
}
Api.JetResetTableSequential(session, table, ResetTableSequentialGrbit.None);
}
}