示例#1
0
		public void AfterDelete(string key, Etag deletedEtag)
		{
			foreach (var behavior in prefetchingBehaviors)
			{
				behavior.AfterDelete(key, deletedEtag);
			}
		}
示例#2
0
		protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
		{
			var totalCount = 0;
			while (true)
			{
				var array = GetAttachments(totalCount, lastEtag);
				if (array.Length == 0)
				{
					var databaseStatistics = await GetStats();
					if (lastEtag == null) lastEtag = Etag.Empty;
					if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
					{
						lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
						ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
						continue;
					}
					ShowProgress("Done with reading attachments, total: {0}", totalCount);
					return lastEtag;
				}
				totalCount += array.Length;
				ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount);
				foreach (var item in array)
				{
					item.WriteTo(jsonWriter);
				}
				lastEtag = Etag.Parse(array.Last().Value<string>("Etag"));
			}
		}
		public void DeleteAttachment(string key, Etag etag)
		{
			Api.JetSetCurrentIndex(session, Files, "by_name");
			Api.MakeKey(session, Files, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
			if (Api.TrySeek(session, Files, SeekGrbit.SeekEQ) == false)
			{
				logger.Debug("Attachment with key '{0}' was not found, and considered deleted", key);
				return;
			}

			var fileEtag = Etag.Parse(Api.RetrieveColumn(session, Files, tableColumnsCache.FilesColumns["etag"]));
			if (fileEtag != etag && etag != null)
			{
				throw new ConcurrencyException("DELETE attempted on attachment '" + key +
					"' using a non current etag")
				{
					ActualETag = fileEtag,
					ExpectedETag = etag
				};
			}

			Api.JetDelete(session, Files);

			if (Api.TryMoveFirst(session, Details))
				Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["attachment_count"], -1);
			logger.Debug("Attachment with key '{0}' was deleted", key);
		}
示例#4
0
		public Etag CalculateSynchronizationEtag(Etag etag, Etag lastProcessedEtag)
		{
			if (etag == null)
			{
				if (lastProcessedEtag != null)
				{
					lock (locker)
					{
						if (currentEtag == null && lastProcessedEtag.CompareTo(synchronizationEtag) != 0)
						{
							synchronizationEtag = lastProcessedEtag;
							PersistSynchronizationState();
						}
					}

					return lastProcessedEtag;
				}

				return Etag.Empty;
			}

			if (lastProcessedEtag == null)
				return Etag.Empty;

			if (etag.CompareTo(lastProcessedEtag) < 0)
				return EtagUtil.Increment(etag, -1);

			return lastProcessedEtag;
		}
示例#5
0
		public void AfterUpdate(string key, Etag etagBeforeUpdate)
		{
			foreach (var behavior in prefetchingBehaviors)
			{
				behavior.AfterUpdate(key, etagBeforeUpdate);
			}
		}
示例#6
0
文件: Lists.cs 项目: cocytus/ravendb
		public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
		{
			Api.JetSetCurrentIndex(session, Lists, "by_name_and_etag");
			Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
			Api.MakeKey(session, Lists, start.TransformToValueForEsentSorting(), MakeKeyGrbit.None);
			if (Api.TrySeek(session, Lists, SeekGrbit.SeekGT) == false)
				yield break;
		
			int count = 0;
			do
			{
				var nameFromDb = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["name"], Encoding.Unicode);
				if (string.Equals(name, nameFromDb, StringComparison.InvariantCultureIgnoreCase) == false)
					yield break;


				var etag = Etag.Parse(Api.RetrieveColumn(session, Lists, tableColumnsCache.ListsColumns["etag"]));
				if (end != null && end.CompareTo(etag) <= 0)
					yield break;

				count++;
				
				using (Stream stream = new BufferedStream(new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"])))
				{
					yield return new ListItem
					{
						Etag = etag,
						Data = stream.ToJObject(),
						Key = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["key"], Encoding.Unicode)
					};
				}
			} while (Api.TryMoveNext(session, Lists) && count < take);

		}
		public void FinishSynchronization(string fileName, SynchronizationReport report, FileSystemInfo sourceFileSystem, Etag sourceFileETag)
		{
			try
			{
				// we want to execute those operation in a single batch but we also have to ensure that
				// Raven/Synchronization/Sources/sourceServerId config is modified only by one finishing synchronization at the same time
				synchronizationFinishLocks.GetOrAdd(sourceFileSystem.Id, new ReaderWriterLockSlim()).EnterWriteLock();

				Storage.Batch(accessor =>
				{
					SaveSynchronizationReport(fileName, accessor, report);
					FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor);

					if (report.Exception == null)
						SaveSynchronizationSourceInformation(sourceFileSystem, sourceFileETag);
				});
			}
			catch (Exception ex)
			{
				Log.ErrorException(string.Format("Failed to finish synchronization of a file '{0}' from {1}", fileName, sourceFileSystem), ex);
			}
			finally
			{
				synchronizationFinishLocks.GetOrAdd(sourceFileSystem.Id, new ReaderWriterLockSlim()).ExitWriteLock();
			}
		}
		public Etag AddAttachment(string key, Etag etag, Stream data, RavenJObject headers)
		{
			Api.JetSetCurrentIndex(session, Files, "by_name");
			Api.MakeKey(session, Files, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
			var isUpdate = Api.TrySeek(session, Files, SeekGrbit.SeekEQ);
			if (isUpdate)
			{
				var existingEtag = Etag.Parse(Api.RetrieveColumn(session, Files, tableColumnsCache.FilesColumns["etag"]));
				if (existingEtag != etag && etag != null)
				{
					throw new ConcurrencyException("PUT attempted on attachment '" + key +
						"' using a non current etag")
					{
						ActualETag = existingEtag,
						ExpectedETag = etag
					};
				}
			}
			else
			{
				if (data == null)
					throw new InvalidOperationException("When adding new attachment, the attachment data must be specified");

				if (Api.TryMoveFirst(session, Details))
					Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["attachment_count"], 1);
			}

			Etag newETag = uuidGenerator.CreateSequentialUuid(UuidType.Attachments);
			using (var update = new Update(session, Files, isUpdate ? JET_prep.Replace : JET_prep.Insert))
			{
				Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["name"], key, Encoding.Unicode);
				if (data != null)
				{
					long written;
					using (var columnStream = new ColumnStream(session, Files, tableColumnsCache.FilesColumns["data"]))
					{
						if (isUpdate)
							columnStream.SetLength(0);
						using (var stream = new BufferedStream(columnStream))
						{
							data.CopyTo(stream);
							written = stream.Position;
							stream.Flush();
						}
					}
					if (written == 0) // empty attachment
					{
						Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["data"], new byte[0]);
					}
				}

				Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], newETag.TransformToValueForEsentSorting());
				Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], headers.ToString(Formatting.None), Encoding.Unicode);

				update.Save();
			}
			logger.Debug("Adding attachment {0}", key);

			return newETag;
		}
示例#9
0
		public IndexingBatch(Etag highestEtagBeforeFiltering)
		{
			HighestEtagBeforeFiltering = highestEtagBeforeFiltering;
			Ids = new List<string>();
			Docs = new List<dynamic>();
			SkipDeleteFromIndex = new List<bool>();
		}
示例#10
0
		public void UpdateSynchronizationState(Etag lowestEtag)
		{
			lock (locker)
			{
				if (UpdateSynchronizationStateInternal(lowestEtag))
					PersistSynchronizationState();
			}
		}
示例#11
0
		private bool UpdateSynchronizationStateInternal(Etag lowestEtag)
		{
			if (currentEtag == null || lowestEtag.CompareTo(currentEtag) < 0)
			{
				currentEtag = lowestEtag;
			}

			return lowestEtag.CompareTo(synchronizationEtag) < 0;
		}
        public Task<IAsyncEnumerator<RavenJObject>> GetDocuments(RavenConnectionStringOptions src, Etag lastEtag, int take)
        {
            const int dummy = 0;
            var enumerator = database.Documents.GetDocumentsAsJson(dummy, Math.Min(Options.BatchSize, take), lastEtag, CancellationToken.None)
                .ToList()
                .Cast<RavenJObject>()
                .GetEnumerator();

            return new CompletedTask<IAsyncEnumerator<RavenJObject>>(new AsyncEnumeratorBridge<RavenJObject>(enumerator));
        }
示例#13
0
		protected override Task<IAsyncEnumerator<RavenJObject>> GetDocuments(Etag lastEtag)
		{
			const int dummy = 0;

			var enumerator = _database.GetDocuments(dummy, SmugglerOptions.BatchSize, lastEtag)
				.ToList()
				.Cast<RavenJObject>()
				.GetEnumerator();

			return new CompletedTask<IAsyncEnumerator<RavenJObject>>(new AsyncEnumeratorBridge<RavenJObject>(enumerator));
		}
示例#14
0
 public StorageExporter(string databaseBaseDirectory, string databaseOutputFile,int batchSize,Etag documentsStartEtag)
 {
     baseDirectory = databaseBaseDirectory;
     outputDirectory = databaseOutputFile;
     var ravenConfiguration = new RavenConfiguration();
     ravenConfiguration.DataDirectory = databaseBaseDirectory;
     ravenConfiguration.Storage.PreventSchemaUpdate = true;
     ravenConfiguration.Storage.SkipConsistencyCheck = true;
     CreateTransactionalStorage(ravenConfiguration);
     BatchSize = batchSize;
     DocumentsStartEtag = documentsStartEtag;
 }
示例#15
0
		private List<JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag)
		{
			var inMemResults = new List<JsonDocument>();
			var nextDocEtag = GetNextDocEtag(etag);
			if (TryGetInMemoryJsonDocuments(nextDocEtag, inMemResults))
				return inMemResults;

			var results =
				GetFutureJsonDocuments(nextDocEtag) ??
				GetJsonDocsFromDisk(etag); // here we _intentionally_ using the current etag, not the next one

			return MergeWithOtherFutureResults(results);
		}
        public Task<IAsyncEnumerator<FileHeader>> GetFiles(Etag lastEtag, int take)
        {
            ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take);

            IEnumerable<FileHeader> enumerable = null;

            filesystem.Storage.Batch(accessor =>
            {
                enumerable = accessor.GetFilesAfter(lastEtag, take);
            });

            return new CompletedTask<IAsyncEnumerator<FileHeader>>(new AsyncEnumeratorBridge<FileHeader>(enumerable.GetEnumerator()));
        }
示例#17
0
		protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag)
		{
			int totalCount = 0;
			while (true)
			{
				RavenJArray attachmentInfo = null;

				await commands.CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag, "GET")
				              .ReadResponseJsonAsync()
				              .ContinueWith(task => attachmentInfo = (RavenJArray) task.Result);

				if (attachmentInfo.Length == 0)
				{
					var databaseStatistics = await GetStats();
					var lastEtagComparable = new ComparableByteArray(lastEtag);
					if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
					{
						lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
						ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag);
						continue;
					}

					ShowProgress("Done with reading attachments, total: {0}", totalCount);
					return lastEtag;
				}

				totalCount += attachmentInfo.Length;
				ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount);
				foreach (var item in attachmentInfo)
				{
					ShowProgress("Downloading attachment: {0}", item.Value<string>("Key"));

					byte[] attachmentData = null;

					await commands.CreateRequest("/static/" + item.Value<string>("Key"), "GET")
					              .ReadResponseBytesAsync()
					              .ContinueWith(task => attachmentData = task.Result);

					new RavenJObject
					{
						{"Data", attachmentData},
						{"Metadata", item.Value<RavenJObject>("Metadata")},
						{"Key", item.Value<string>("Key")}
					}
						.WriteTo(jsonWriter);
				}

				lastEtag = Etag.Parse(attachmentInfo.Last().Value<string>("Etag"));
			}
		}
示例#18
0
		public Etag GetSynchronizationEtag()
		{
			lock (locker)
			{
				var etag = currentEtag;
				if (etag != null)
				{
					PersistSynchronizationState();
					synchronizationEtag = currentEtag;
					currentEtag = null;
				}

				return etag;
			}
		}
示例#19
0
        private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter)
        {
            var bufferStream = new BufferedStream(stream, 1024 * 64);
            using (var cts = new CancellationTokenSource())
            using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
            using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
            {
                writer.WriteStartObject();
                writer.WritePropertyName("Results");
                writer.WriteStartArray();

                Action<JsonDocument> addDocument = doc =>
                {
                    timeout.Delay();
                    doc.ToJson().WriteTo(writer);
                    writer.WriteRaw(Environment.NewLine);
                };

                Database.TransactionalStorage.Batch(accessor =>
                {
                    // we may be sending a LOT of documents to the user, and most 
                    // of them aren't going to be relevant for other ops, so we are going to skip
                    // the cache for that, to avoid filling it up very quickly
                    using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
                    {
                        if (string.IsNullOrEmpty(startsWith))
                        {
                            Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, addDocument);
                        }
                        else
                        {
                            var nextPageStartInternal = nextPageStart;

                            Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, addDocument, skipAfter: skipAfter);

                            nextPageStart = nextPageStartInternal;
                        }
                    }
                });

                writer.WriteEndArray();
                writer.WritePropertyName("NextPageStart");
                writer.WriteValue(nextPageStart);
                writer.WriteEndObject();
                writer.Flush();
                bufferStream.Flush();
            }
        }
示例#20
0
		public List<JsonDocument> GetDocumentsBatchFrom(Etag etag)
		{
			var results = GetDocsFromBatchWithPossibleDuplicates(etag);
			// a single doc may appear multiple times, if it was updated while we were fetching things, 
			// so we have several versions of the same doc loaded, this will make sure that we will only  
			// take one of them.
			var ids = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
			for (int i = results.Count - 1; i >= 0; i--)
			{
				if(CanBeConsideredAsDuplicate(results[i]) && ids.Add(results[i].Key) == false)
				{
					results.RemoveAt(i);
				}
			}
			return results;
		}
示例#21
0
		public void RemoveAllBefore(string name, Etag etag)
		{
			var comparable = new ComparableByteArray(etag);
			var results = storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
			{
				{"name", name},
				{"etag", Guid.Empty.ToByteArray()}
			})
				.TakeWhile(x => String.Equals(x.Value<string>("name"), name, StringComparison.OrdinalIgnoreCase) &&
				                comparable.CompareTo(x.Value<byte[]>("etag")) >= 0);

			foreach (var result in results)
			{
				storage.Lists.Remove(result);
			}
		}
 public Task<Etag> ExportDocumentsDeletion(JsonTextWriter jsonWriter, Etag startDocsEtag, Etag maxEtag)
 {
     var lastEtag = startDocsEtag;
     database.TransactionalStorage.Batch(accessor =>
     {
         foreach (var listItem in accessor.Lists.Read(Constants.RavenPeriodicExportsDocsTombstones, startDocsEtag, maxEtag, int.MaxValue))
         {
             var o = new RavenJObject
             {
                 {"Key", listItem.Key}
             };
             o.WriteTo(jsonWriter);
             lastEtag = listItem.Etag;
         }
     });
     return new CompletedTask<Etag>(lastEtag);
 }
示例#23
0
		public void RemoveAllBefore(string name, Etag etag)
		{
			Api.JetSetCurrentIndex(session, Lists, "by_name_and_etag");
			Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
			Api.MakeKey(session, Lists, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.None);
			if (Api.TrySeek(session, Lists, SeekGrbit.SeekLE) == false)
				return;
			do
			{
				var nameFromDb = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["name"], Encoding.Unicode);
				if (string.Equals(name, nameFromDb, StringComparison.OrdinalIgnoreCase) == false)
					break;

				Api.JetDelete(session, Lists);

			} while (Api.TryMovePrevious(session, Lists));

		}
示例#24
0
		protected override RavenJArray GetDocuments(Etag lastEtag)
		{
			int retries = retriesCount;
			while (true)
			{
				try
				{
					RavenJArray documents = null;
					var request = CreateRequest("/docs?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag);
					request.ExecuteRequest(reader => documents = RavenJArray.Load(new JsonTextReader(reader)));
					return documents;
				}
				catch (Exception e)
				{
					if (retries-- == 0)
						throw;
					LastRequestErrored = true;
					ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e, retriesCount);
				}
			}
		}
示例#25
0
		private void SaveSynchronizationSourceInformation(FileSystemInfo sourceFileSystem, Etag lastSourceEtag)
		{
			var lastSynchronizationInformation = GetLastSynchronization(sourceFileSystem.Id);
			if (EtagUtil.IsGreaterThan(lastSynchronizationInformation.LastSourceFileEtag, lastSourceEtag))
			{
				return;
			}

			var synchronizationSourceInfo = new SourceSynchronizationInformation
			{
				LastSourceFileEtag = lastSourceEtag,
				SourceServerUrl = sourceFileSystem.Url,
				DestinationServerId = Storage.Id
			};

			var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceFileSystem.Id;

			Storage.Batch(accessor => accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo)));
			

			Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceFileSystem.Url, sourceFileSystem.Id);
		}
示例#26
0
		public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
		{
			return storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject
			{
				{ "name", name },
				{ "etag", start.ToByteArray() }
			})
			.TakeWhile(x=> 
				StringComparer.OrdinalIgnoreCase.Equals(x.Value<string>("name"), name) &&
				(end == null || end.CompareTo(Etag.Parse(x.Value<byte[]>("etag"))) > 0)
				)
			.Take(take)
			.Select(result =>
			{
				var readResult = storage.Lists.Read(result);
				return new ListItem
				{
					Data = readResult.Data().ToJObject(),
					Etag = Etag.Parse(readResult.Key.Value<byte[]>("etag")),
					Key = readResult.Key.Value<string>("key")
				};
			});
		}
示例#27
0
		protected override bool IsIndexStale(IndexStats indexesStat, Etag synchronizationEtag, IStorageActionsAccessor actions, bool isIdle, Reference<bool> onlyFoundIdleWork)
		{
			if (indexesStat.LastIndexedEtag.CompareTo(synchronizationEtag) > 0)
				return true;

			var isStale = actions.Staleness.IsMapStale(indexesStat.Name);
			var indexingPriority = indexesStat.Priority;
			if (isStale == false)
				return false;

			if (indexingPriority == IndexingPriority.None)
				return true;

			if (indexingPriority.HasFlag(IndexingPriority.Normal))
			{
				onlyFoundIdleWork.Value = false;
				return true;
			}

			if (indexingPriority.HasFlag(IndexingPriority.Disabled))
				return false;

			if (isIdle == false)
				return false; // everything else is only valid on idle runs

			if (indexingPriority.HasFlag(IndexingPriority.Idle))
				return true;

			if (indexingPriority.HasFlag(IndexingPriority.Abandoned))
			{
				var timeSinceLastIndexing = (SystemTime.UtcNow - indexesStat.LastIndexingTime);

				return (timeSinceLastIndexing > context.Configuration.TimeToWaitBeforeRunningAbandonedIndexes);
			}

			throw new InvalidOperationException("Unknown indexing priority for index " + indexesStat.Name + ": " + indexesStat.Priority);
		}
示例#28
0
 public Etag ToEtag()
 {
     return(Etag.Parse(inner));
 }
示例#29
0
		protected override Task<IAsyncEnumerator<RavenJObject>> GetDocuments(Etag lastEtag)
		{
			return commands.StreamDocsAsync(lastEtag);
		}
示例#30
0
		private List<JsonDocument> GetDocsFromBatchWithPossibleDuplicates(Etag etag)
		{
			var result = new List<JsonDocument>();
			bool docsLoaded;
			do
			{
				var nextEtagToIndex = GetNextDocEtag(etag);
				var firstEtagInQueue = prefetchingQueue.NextDocumentETag();

				if (nextEtagToIndex != firstEtagInQueue)
				{
					if (TryLoadDocumentsFromFutureBatches(nextEtagToIndex) == false)
					{
						LoadDocumentsFromDisk(etag, firstEtagInQueue); // here we _intentionally_ use the current etag, not the next one
					}
				}

				docsLoaded = TryGetDocumentsFromQueue(nextEtagToIndex, ref result);

				if (docsLoaded)
					etag = result[result.Count - 1].Etag;

			} while (result.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && docsLoaded);
			

			return result;
		}
示例#31
0
 public ComparableByteArray(Etag etag) : this(etag.ToByteArray())
 {
 }
示例#32
0
		private Etag SkipUpdatedEtags(Etag nextEtag)
		{
			while (updatedDocuments.Any(x => x.Value.Contains(nextEtag)))
			{
				nextEtag = EtagUtil.Increment(nextEtag, 1);
			}

			return nextEtag;
		}