public IEnumerable <string> QueryDocumentIds(string index, IndexQuery query, CancellationTokenSource tokenSource, out bool stale)
{
var queryStat = AddToCurrentlyRunningQueryList(index, query, tokenSource);
try
{
bool isStale = false;
HashSet <string> loadedIds = null;
TransactionalStorage.Batch(
actions =>
{
var definition = IndexDefinitionStorage.GetIndexDefinition(index);
if (definition == null)
{
throw new ArgumentException("specified index definition was not found", "index");
}
isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null);
if (isStale == false && query.Cutoff == null)
{
var indexInstance = Database.IndexStorage.GetIndexInstance(index);
isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
if (isStale && actions.Staleness.IsIndexStaleByTask(definition.IndexId, query.Cutoff) == false &&
actions.Staleness.IsReduceStale(definition.IndexId) == false)
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
throw new ArgumentException("specified index definition was not found", "index");
}
var forEntityNames = viewGenerator.ForEntityNames.ToList();
var lastIndexedEtag = actions.Indexing.GetIndexStats(definition.IndexId).LastIndexedEtag;
if (Database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
{
isStale = false;
}
}
var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
loadedIds = new HashSet <string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, tokenSource.Token)
select queryResult.Key);
});
stale = isStale;
return(loadedIds);
}
finally
{
RemoveFromCurrentlyRunningQueryList(index, queryStat);
}
}
public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
JsonDocumentMetadata document = null;
if (transactionInformation == null ||
Database.InFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)
{
TransactionalStorage.Batch(actions =>
{
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocumentMetadata>(transactionInformation, key);
document = actions.Documents.DocumentMetadataByKey(key);
if (nonAuthoritativeInformationBehavior != null)
{
document = nonAuthoritativeInformationBehavior(document);
}
});
}
JsonDocument.EnsureIdInMetadata(document);
return(new DocumentRetriever(null, null, Database.ReadTriggers)
.ProcessReadVetoes(document, transactionInformation, ReadOperation.Load));
}
public EsentInFlightTransactionalState(TransactionalStorage storage, CommitTransactionGrbit txMode, Func <string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> databasePut, Func <string, Etag, TransactionInformation, bool> databaseDelete)
: base(databasePut, databaseDelete)
{
this.storage = storage;
this.txMode = txMode;
timer = new Timer(CleanupOldTransactions, null, TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(5));
}
public void RecoverIndexingErrors()
{
var storedIndexingErrors = new List<ListItem>();
TransactionalStorage.Batch(accessor =>
{
foreach (var indexName in IndexDefinitionStorage.IndexNames)
{
storedIndexingErrors.AddRange(accessor.Lists.Read("Raven/Indexing/Errors/" + indexName, Etag.Empty, null, 5000));
}
});
if(storedIndexingErrors.Count == 0)
return;
var errors = storedIndexingErrors.Select(x => x.Data.JsonDeserialization<IndexingError>()).OrderBy(x => x.Timestamp);
foreach (var error in errors)
{
indexingErrors.Enqueue(error);
}
TransactionalStorage.Batch(accessor =>
{
while (indexingErrors.Count > 50)
{
IndexingError error;
if (indexingErrors.TryDequeue(out error) == false)
continue;
accessor.Lists.Remove("Raven/Indexing/Errors/" + error.IndexName, error.Id.ToString(CultureInfo.InvariantCulture));
}
});
errorsCounter = errors.Max(x => x.Id);
}
public DocumentStorageActions(
JET_INSTANCE instance,
string database,
TableColumnsCache tableColumnsCache,
OrderedPartCollection <AbstractDocumentCodec> documentCodecs,
IUuidGenerator uuidGenerator,
IDocumentCacher cacher,
TransactionalStorage transactionalStorage)
{
this.tableColumnsCache = tableColumnsCache;
this.documentCodecs = documentCodecs;
this.uuidGenerator = uuidGenerator;
this.cacher = cacher;
this.transactionalStorage = transactionalStorage;
try
{
session = new Session(instance);
transaction = new Transaction(session);
Api.JetOpenDatabase(session, database, null, out dbid, OpenDatabaseGrbit.None);
}
catch (Exception)
{
Dispose();
throw;
}
}
public override int Read(byte[] buffer, int offset, int count)
{
if (currentOffset >= Length)
{
return(0);
}
var innerBuffer = new byte[StorageConstants.MaxPageSize];
var pageOffset = currentPageFrameOffset;
var length = 0L;
var startingOffset = currentOffset;
foreach (var page in fileAndPages.Pages)
{
if (pageOffset <= currentOffset && currentOffset < pageOffset + page.Size)
{
var pageLength = 0;
TransactionalStorage.Batch(accessor => pageLength = accessor.ReadPage(page.Id, innerBuffer));
var sourceIndex = currentOffset - pageOffset;
length = Math.Min(innerBuffer.Length - sourceIndex,
Math.Min(pageLength, Math.Min(buffer.Length - offset, Math.Min(pageLength - sourceIndex, count))));
Array.Copy(innerBuffer, sourceIndex, buffer, offset, length);
break;
}
pageOffset += page.Size;
}
MovePageFrame(currentOffset + length);
return(Convert.ToInt32(currentOffset - startingOffset));
}
public void StartBackup(string backupDestinationDirectory, bool incrementalBackup, DatabaseDocument databaseDocument)
{
if (databaseDocument == null)
{
throw new ArgumentNullException("databaseDocument");
}
var document = Database.Documents.Get(BackupStatus.RavenBackupStatusDocumentKey);
if (document != null)
{
var backupStatus = document.DataAsJson.JsonDeserialization <BackupStatus>();
if (backupStatus.IsRunning)
{
throw new InvalidOperationException("Backup is already running");
}
}
if (incrementalBackup &&
TransactionalStorage is Raven.Storage.Voron.TransactionalStorage &&
Database.Configuration.Storage.AllowIncrementalBackups == false)
{
throw new InvalidOperationException($"In order to run incremental backups using Voron you must have the appropriate setting key ({RavenConfiguration.GetKey(x => x.Storage.AllowIncrementalBackups)}) set to true");
}
Database.Documents.Put(BackupStatus.RavenBackupStatusDocumentKey, null, RavenJObject.FromObject(new BackupStatus
{
Started = SystemTime.UtcNow,
IsRunning = true,
}), new RavenJObject(), null);
Database.IndexStorage.FlushMapIndexes();
Database.IndexStorage.FlushReduceIndexes();
TransactionalStorage.StartBackupOperation(Database, backupDestinationDirectory, incrementalBackup, databaseDocument);
}
public void DeleteIndex(string name)
{
using (IndexDefinitionStorage.TryRemoveIndexContext())
{
var instance = IndexDefinitionStorage.GetIndexDefinition(name);
if (instance == null)
{
return;
}
// Set up a flag to signal that this is something we're doing
TransactionalStorage.Batch(actions => actions.Lists.Set("Raven/Indexes/PendingDeletion", instance.IndexId.ToString(CultureInfo.InvariantCulture), (RavenJObject.FromObject(new
{
TimeOfOriginalDeletion = SystemTime.UtcNow,
instance.IndexId,
IndexName = instance.Name
})), UuidType.Tasks));
// Delete the main record synchronously
IndexDefinitionStorage.RemoveIndex(name);
Database.IndexStorage.DeleteIndex(instance.IndexId);
WorkContext.ClearErrorsFor(name);
// And delete the data in the background
StartDeletingIndexDataAsync(instance.IndexId, name);
// We raise the notification now because as far as we're concerned it is done *now*
TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification
{
Name = name,
Type = IndexChangeTypes.IndexRemoved,
}));
}
}
public void Delete(string key, Guid?etag, TransactionInformation transactionInformation)
{
TransactionalStorage.Batch(actions =>
{
if (transactionInformation == null)
{
AssertDeleteOperationNotVetoed(key, transactionInformation);
DeleteTriggers.Apply(trigger => trigger.OnDelete(key, transactionInformation));
JObject metadata;
if (actions.DeleteDocument(key, etag, out metadata))
{
AddIndexingTask(actions, metadata, () => new RemoveFromIndexTask {
Keys = new[] { key }
});
DeleteTriggers.Apply(trigger => trigger.AfterDelete(key, transactionInformation));
}
}
else
{
actions.DeleteDocumentInTransaction(transactionInformation, key, etag);
}
workContext.ShouldNotifyAboutWork();
});
TransactionalStorage
.ExecuteImmediatelyOrRegisterForSyncronization(() => DeleteTriggers.Apply(trigger => trigger.AfterCommit(key)));
}
public EsentInFlightTransactionalState(TransactionalStorage storage, CommitTransactionGrbit txMode, Func<string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> databasePut, Func<string, Etag, TransactionInformation, bool> databaseDelete)
: base(databasePut, databaseDelete)
{
this.storage = storage;
this.txMode = txMode;
timer = new Timer(CleanupOldTransactions, null, TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(5));
}
internal Task StartDeletingIndexDataAsync(int id, string indexName)
{
//remove the header information in a sync process
TransactionalStorage.Batch(actions => actions.Indexing.PrepareIndexForDeletion(id));
var deleteIndexTask = Task.Run(() =>
{
Debug.Assert(Database.IndexStorage != null);
Database.IndexStorage.DeleteIndexData(id); // Data can take a while
TransactionalStorage.Batch(actions =>
{
// And Esent data can take a while too
actions.Indexing.DeleteIndex(id, WorkContext.CancellationToken);
if (WorkContext.CancellationToken.IsCancellationRequested)
{
return;
}
actions.Lists.Remove("Raven/Indexes/PendingDeletion", id.ToString(CultureInfo.InvariantCulture));
});
});
long taskId;
Database.Tasks.AddTask(deleteIndexTask, null, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.IndexDeleteOperation,
Payload = indexName
}, out taskId);
deleteIndexTask.ContinueWith(_ => Database.Tasks.RemoveTask(taskId));
return(deleteIndexTask);
}
public EsentInFlightTransactionalState(DocumentDatabase database, TransactionalStorage storage, CommitTransactionGrbit txMode, Func<string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> databasePut, Func<string, Etag, TransactionInformation, bool> databaseDelete)
: base(databasePut, databaseDelete)
{
_database = database;
this.storage = storage;
this.txMode = txMode;
timer = database.TimerManager.NewTimer(CleanupOldTransactions, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30));
}
public EsentInFlightTransactionalState(DocumentDatabase database, TransactionalStorage storage, CommitTransactionGrbit txMode, Func <string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> databasePut, Func <string, Etag, TransactionInformation, bool> databaseDelete)
: base(databasePut, databaseDelete, database.StartupTasks.OfType <ReplicationTask>().FirstOrDefault() != null)
{
_database = database;
this.storage = storage;
this.txMode = txMode;
timer = database.TimerManager.NewTimer(CleanupOldTransactions, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30));
}
public void Rollback(Guid txId)
{
TransactionalStorage.Batch(actions =>
{
actions.RollbackTransaction(txId);
workContext.ShouldNotifyAboutWork();
});
}
public ITransactionalStorage NewTransactionalStorage()
{
var newTransactionalStorage = new TransactionalStorage(new RavenConfiguration
{
DataDirectory = "test",
}, () => { });
newTransactionalStorage.Initialize();
return newTransactionalStorage;
}
public ViewStorage()
{
transactionalStorage = new TransactionalStorage(new RavenConfiguration
{
DataDirectory = "raven.db.test.esent",
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = true
}, () => { });
transactionalStorage.Initialize();
}
public ITransactionalStorage NewTransactionalStorage()
{
var newTransactionalStorage = new TransactionalStorage(new RavenConfiguration
{
DataDirectory = "test",
}, () => { })
{
DocumentCodecs = new AbstractDocumentCodec[0]
};
newTransactionalStorage.Initialize(new DummyUuidGenerator());
return newTransactionalStorage;
}
public MuninInFlightTransactionalState(TransactionalStorage storage, Func<string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> databasePut, Func<string, Etag, TransactionInformation, bool> databaseDelete) : base(databasePut, databaseDelete)
{
this.storage = storage;
}
