public void AfterDelete(string key, Etag deletedEtag)
{
foreach (var behavior in prefetchingBehaviors)
{
behavior.AfterDelete(key, deletedEtag);
}
}
protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
while (true)
{
var array = GetAttachments(totalCount, lastEtag);
if (array.Length == 0)
{
var databaseStatistics = await GetStats();
if (lastEtag == null) lastEtag = Etag.Empty;
if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += array.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount);
foreach (var item in array)
{
item.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(array.Last().Value<string>("Etag"));
}
}
public void DeleteAttachment(string key, Etag etag)
{
Api.JetSetCurrentIndex(session, Files, "by_name");
Api.MakeKey(session, Files, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekEQ) == false)
{
logger.Debug("Attachment with key '{0}' was not found, and considered deleted", key);
return;
}
var fileEtag = Etag.Parse(Api.RetrieveColumn(session, Files, tableColumnsCache.FilesColumns["etag"]));
if (fileEtag != etag && etag != null)
{
throw new ConcurrencyException("DELETE attempted on attachment '" + key +
"' using a non current etag")
{
ActualETag = fileEtag,
ExpectedETag = etag
};
}
Api.JetDelete(session, Files);
if (Api.TryMoveFirst(session, Details))
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["attachment_count"], -1);
logger.Debug("Attachment with key '{0}' was deleted", key);
}
public Etag CalculateSynchronizationEtag(Etag etag, Etag lastProcessedEtag)
{
if (etag == null)
{
if (lastProcessedEtag != null)
{
lock (locker)
{
if (currentEtag == null && lastProcessedEtag.CompareTo(synchronizationEtag) != 0)
{
synchronizationEtag = lastProcessedEtag;
PersistSynchronizationState();
}
}
return lastProcessedEtag;
}
return Etag.Empty;
}
if (lastProcessedEtag == null)
return Etag.Empty;
if (etag.CompareTo(lastProcessedEtag) < 0)
return EtagUtil.Increment(etag, -1);
return lastProcessedEtag;
}
public void AfterUpdate(string key, Etag etagBeforeUpdate)
{
foreach (var behavior in prefetchingBehaviors)
{
behavior.AfterUpdate(key, etagBeforeUpdate);
}
}
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
{
Api.JetSetCurrentIndex(session, Lists, "by_name_and_etag");
Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Lists, start.TransformToValueForEsentSorting(), MakeKeyGrbit.None);
if (Api.TrySeek(session, Lists, SeekGrbit.SeekGT) == false)
yield break;
int count = 0;
do
{
var nameFromDb = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["name"], Encoding.Unicode);
if (string.Equals(name, nameFromDb, StringComparison.InvariantCultureIgnoreCase) == false)
yield break;
var etag = Etag.Parse(Api.RetrieveColumn(session, Lists, tableColumnsCache.ListsColumns["etag"]));
if (end != null && end.CompareTo(etag) <= 0)
yield break;
count++;
using (Stream stream = new BufferedStream(new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"])))
{
yield return new ListItem
{
Etag = etag,
Data = stream.ToJObject(),
Key = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["key"], Encoding.Unicode)
};
}
} while (Api.TryMoveNext(session, Lists) && count < take);
}
public void FinishSynchronization(string fileName, SynchronizationReport report, FileSystemInfo sourceFileSystem, Etag sourceFileETag)
{
try
{
// we want to execute those operation in a single batch but we also have to ensure that
// Raven/Synchronization/Sources/sourceServerId config is modified only by one finishing synchronization at the same time
synchronizationFinishLocks.GetOrAdd(sourceFileSystem.Id, new ReaderWriterLockSlim()).EnterWriteLock();
Storage.Batch(accessor =>
{
SaveSynchronizationReport(fileName, accessor, report);
FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor);
if (report.Exception == null)
SaveSynchronizationSourceInformation(sourceFileSystem, sourceFileETag);
});
}
catch (Exception ex)
{
Log.ErrorException(string.Format("Failed to finish synchronization of a file '{0}' from {1}", fileName, sourceFileSystem), ex);
}
finally
{
synchronizationFinishLocks.GetOrAdd(sourceFileSystem.Id, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
public Etag AddAttachment(string key, Etag etag, Stream data, RavenJObject headers)
{
Api.JetSetCurrentIndex(session, Files, "by_name");
Api.MakeKey(session, Files, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdate = Api.TrySeek(session, Files, SeekGrbit.SeekEQ);
if (isUpdate)
{
var existingEtag = Etag.Parse(Api.RetrieveColumn(session, Files, tableColumnsCache.FilesColumns["etag"]));
if (existingEtag != etag && etag != null)
{
throw new ConcurrencyException("PUT attempted on attachment '" + key +
"' using a non current etag")
{
ActualETag = existingEtag,
ExpectedETag = etag
};
}
}
else
{
if (data == null)
throw new InvalidOperationException("When adding new attachment, the attachment data must be specified");
if (Api.TryMoveFirst(session, Details))
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["attachment_count"], 1);
}
Etag newETag = uuidGenerator.CreateSequentialUuid(UuidType.Attachments);
using (var update = new Update(session, Files, isUpdate ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["name"], key, Encoding.Unicode);
if (data != null)
{
long written;
using (var columnStream = new ColumnStream(session, Files, tableColumnsCache.FilesColumns["data"]))
{
if (isUpdate)
columnStream.SetLength(0);
using (var stream = new BufferedStream(columnStream))
{
data.CopyTo(stream);
written = stream.Position;
stream.Flush();
}
}
if (written == 0) // empty attachment
{
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["data"], new byte[0]);
}
}
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], newETag.TransformToValueForEsentSorting());
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], headers.ToString(Formatting.None), Encoding.Unicode);
update.Save();
}
logger.Debug("Adding attachment {0}", key);
return newETag;
}
public IndexingBatch(Etag highestEtagBeforeFiltering)
{
HighestEtagBeforeFiltering = highestEtagBeforeFiltering;
Ids = new List<string>();
Docs = new List<dynamic>();
SkipDeleteFromIndex = new List<bool>();
}
public void UpdateSynchronizationState(Etag lowestEtag)
{
lock (locker)
{
if (UpdateSynchronizationStateInternal(lowestEtag))
PersistSynchronizationState();
}
}
private bool UpdateSynchronizationStateInternal(Etag lowestEtag)
{
if (currentEtag == null || lowestEtag.CompareTo(currentEtag) < 0)
{
currentEtag = lowestEtag;
}
return lowestEtag.CompareTo(synchronizationEtag) < 0;
}
public Task<IAsyncEnumerator<RavenJObject>> GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take)
{
const int dummy = 0;
var enumerator = database.Documents.GetDocumentsAsJson(dummy, Math.Min(Options.BatchSize, take), lastEtag, CancellationToken.None)
.ToList()
.Cast<RavenJObject>()
.GetEnumerator();
return new CompletedTask<IAsyncEnumerator<RavenJObject>>(new AsyncEnumeratorBridge<RavenJObject>(enumerator));
}
protected override Task<IAsyncEnumerator<RavenJObject>> GetDocuments(Etag lastEtag)
{
const int dummy = 0;
var enumerator = _database.GetDocuments(dummy, SmugglerOptions.BatchSize, lastEtag)
.ToList()
.Cast<RavenJObject>()
.GetEnumerator();
return new CompletedTask<IAsyncEnumerator<RavenJObject>>(new AsyncEnumeratorBridge<RavenJObject>(enumerator));
}
public StorageExporter(string databaseBaseDirectory, string databaseOutputFile,int batchSize,Etag documentsStartEtag)
{
baseDirectory = databaseBaseDirectory;
outputDirectory = databaseOutputFile;
var ravenConfiguration = new RavenConfiguration();
ravenConfiguration.DataDirectory = databaseBaseDirectory;
ravenConfiguration.Storage.PreventSchemaUpdate = true;
ravenConfiguration.Storage.SkipConsistencyCheck = true;
CreateTransactionalStorage(ravenConfiguration);
BatchSize = batchSize;
DocumentsStartEtag = documentsStartEtag;
}
private List<JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag)
{
var inMemResults = new List<JsonDocument>();
var nextDocEtag = GetNextDocEtag(etag);
if (TryGetInMemoryJsonDocuments(nextDocEtag, inMemResults))
return inMemResults;
var results =
GetFutureJsonDocuments(nextDocEtag) ??
GetJsonDocsFromDisk(etag); // here we _intentionally_ using the current etag, not the next one
return MergeWithOtherFutureResults(results);
}
public Task<IAsyncEnumerator<FileHeader>> GetFiles(Etag lastEtag, int take)
{
ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take);
IEnumerable<FileHeader> enumerable = null;
filesystem.Storage.Batch(accessor =>
{
enumerable = accessor.GetFilesAfter(lastEtag, take);
});
return new CompletedTask<IAsyncEnumerator<FileHeader>>(new AsyncEnumeratorBridge<FileHeader>(enumerable.GetEnumerator()));
}
protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
{
int totalCount = 0;
while (true)
{
RavenJArray attachmentInfo = null;
await commands.CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag, "GET")
.ReadResponseJsonAsync()
.ContinueWith(task => attachmentInfo = (RavenJArray) task.Result);
if (attachmentInfo.Length == 0)
{
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return lastEtag;
}
totalCount += attachmentInfo.Length;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
foreach (var item in attachmentInfo)
{
ShowProgress("Downloading attachment: {0}", item.Value<string>("Key"));
byte[] attachmentData = null;
await commands.CreateRequest("/static/" + item.Value<string>("Key"), "GET")
.ReadResponseBytesAsync()
.ContinueWith(task => attachmentData = task.Result);
new RavenJObject
{
{"Data", attachmentData},
{"Metadata", item.Value<RavenJObject>("Metadata")},
{"Key", item.Value<string>("Key")}
}
.WriteTo(jsonWriter);
}
lastEtag = Etag.Parse(attachmentInfo.Last().Value<string>("Etag"));
}
}
public Etag GetSynchronizationEtag()
{
lock (locker)
{
var etag = currentEtag;
if (etag != null)
{
PersistSynchronizationState();
synchronizationEtag = currentEtag;
currentEtag = null;
}
return etag;
}
}
private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter)
{
var bufferStream = new BufferedStream(stream, 1024 * 64);
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
Action<JsonDocument> addDocument = doc =>
{
timeout.Delay();
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
};
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
if (string.IsNullOrEmpty(startsWith))
{
Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, addDocument);
}
else
{
var nextPageStartInternal = nextPageStart;
Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, addDocument, skipAfter: skipAfter);
nextPageStart = nextPageStartInternal;
}
}
});
writer.WriteEndArray();
writer.WritePropertyName("NextPageStart");
writer.WriteValue(nextPageStart);
writer.WriteEndObject();
writer.Flush();
bufferStream.Flush();
}
}
public List<JsonDocument> GetDocumentsBatchFrom(Etag etag)
{
var results = GetDocsFromBatchWithPossibleDuplicates(etag);
// a single doc may appear multiple times, if it was updated while we were fetching things,
// so we have several versions of the same doc loaded, this will make sure that we will only
// take one of them.
var ids = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (int i = results.Count - 1; i >= 0; i--)
{
if(CanBeConsideredAsDuplicate(results[i]) && ids.Add(results[i].Key) == false)
{
results.RemoveAt(i);
}
}
return results;
}
public void RemoveAllBefore(string name, Etag etag)
{
var comparable = new ComparableByteArray(etag);
var results = storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
{
{"name", name},
{"etag", Guid.Empty.ToByteArray()}
})
.TakeWhile(x => String.Equals(x.Value<string>("name"), name, StringComparison.OrdinalIgnoreCase) &&
comparable.CompareTo(x.Value<byte[]>("etag")) >= 0);
foreach (var result in results)
{
storage.Lists.Remove(result);
}
}
public Task<Etag> ExportDocumentsDeletion(JsonTextWriter jsonWriter, Etag startDocsEtag, Etag maxEtag)
{
var lastEtag = startDocsEtag;
database.TransactionalStorage.Batch(accessor =>
{
foreach (var listItem in accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, startDocsEtag, maxEtag, int.MaxValue))
{
var o = new RavenJObject
{
{"Key", listItem.Key}
};
o.WriteTo(jsonWriter);
lastEtag = listItem.Etag;
}
});
return new CompletedTask<Etag>(lastEtag);
}
public void RemoveAllBefore(string name, Etag etag)
{
Api.JetSetCurrentIndex(session, Lists, "by_name_and_etag");
Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Lists, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.None);
if (Api.TrySeek(session, Lists, SeekGrbit.SeekLE) == false)
return;
do
{
var nameFromDb = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["name"], Encoding.Unicode);
if (string.Equals(name, nameFromDb, StringComparison.OrdinalIgnoreCase) == false)
break;
Api.JetDelete(session, Lists);
} while (Api.TryMovePrevious(session, Lists));
}
protected override RavenJArray GetDocuments(Etag lastEtag)
{
int retries = retriesCount;
while (true)
{
try
{
RavenJArray documents = null;
var request = CreateRequest("/docs?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag);
request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
return documents;
}
catch (Exception e)
{
if (retries-- == 0)
throw;
LastRequestErrored = true;
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e, retriesCount);
}
}
}
private void SaveSynchronizationSourceInformation(FileSystemInfo sourceFileSystem, Etag lastSourceEtag)
{
var lastSynchronizationInformation = GetLastSynchronization(sourceFileSystem.Id);
if (EtagUtil.IsGreaterThan(lastSynchronizationInformation.LastSourceFileEtag, lastSourceEtag))
{
return;
}
var synchronizationSourceInfo = new SourceSynchronizationInformation
{
LastSourceFileEtag = lastSourceEtag,
SourceServerUrl = sourceFileSystem.Url,
DestinationServerId = Storage.Id
};
var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceFileSystem.Id;
Storage.Batch(accessor => accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo)));
Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceFileSystem.Url, sourceFileSystem.Id);
}
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
{
return storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
{
{ "name", name },
{ "etag", start.ToByteArray() }
})
.TakeWhile(x=>
StringComparer.OrdinalIgnoreCase.Equals(x.Value<string>("name"), name) &&
(end == null || end.CompareTo(Etag.Parse(x.Value<byte[]>("etag"))) > 0)
)
.Take(take)
.Select(result =>
{
var readResult = storage.Lists.Read(result);
return new ListItem
{
Data = readResult.Data().ToJObject(),
Etag = Etag.Parse(readResult.Key.Value<byte[]>("etag")),
Key = readResult.Key.Value<string>("key")
};
});
}
protected override bool IsIndexStale(IndexStats indexesStat, Etag synchronizationEtag, IStorageActionsAccessor actions, bool isIdle, Reference<bool> onlyFoundIdleWork)
{
if (indexesStat.LastIndexedEtag.CompareTo(synchronizationEtag) > 0)
return true;
var isStale = actions.Staleness.IsMapStale(indexesStat.Name);
var indexingPriority = indexesStat.Priority;
if (isStale == false)
return false;
if (indexingPriority == IndexingPriority.None)
return true;
if (indexingPriority.HasFlag(IndexingPriority.Normal))
{
onlyFoundIdleWork.Value = false;
return true;
}
if (indexingPriority.HasFlag(IndexingPriority.Disabled))
return false;
if (isIdle == false)
return false; // everything else is only valid on idle runs
if (indexingPriority.HasFlag(IndexingPriority.Idle))
return true;
if (indexingPriority.HasFlag(IndexingPriority.Abandoned))
{
var timeSinceLastIndexing = (SystemTime.UtcNow - indexesStat.LastIndexingTime);
return (timeSinceLastIndexing > context.Configuration.TimeToWaitBeforeRunningAbandonedIndexes);
}
throw new InvalidOperationException("Unknown indexing priority for index " + indexesStat.Name + ": " + indexesStat.Priority);
}
public Etag ToEtag()
{
return(Etag.Parse(inner));
}
protected override Task<IAsyncEnumerator<RavenJObject>> GetDocuments(Etag lastEtag)
{
return commands.StreamDocsAsync(lastEtag);
}
private List<JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag)
{
var result = new List<JsonDocument>();
bool docsLoaded;
do
{
var nextEtagToIndex = GetNextDocEtag(etag);
var firstEtagInQueue = prefetchingQueue.NextDocumentETag();
if (nextEtagToIndex != firstEtagInQueue)
{
if (TryLoadDocumentsFromFutureBatches(nextEtagToIndex) == false)
{
LoadDocumentsFromDisk(etag, firstEtagInQueue); // here we _intentionally_ use the current etag, not the next one
}
}
docsLoaded = TryGetDocumentsFromQueue(nextEtagToIndex, ref result);
if (docsLoaded)
etag = result[result.Count - 1].Etag;
} while (result.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && docsLoaded);
return result;
}
public ComparableByteArray(Etag etag) : this(etag.ToByteArray())
{
}
private Etag SkipUpdatedEtags(Etag nextEtag)
{
while (updatedDocuments.Any(x => x.Value.Contains(nextEtag)))
{
nextEtag = EtagUtil.Increment(nextEtag, 1);
}
return nextEtag;
}