Beispiel #1
0
        private void WriteToBuffer(Stream bufferedStream, out long bytesWritten)
        {
            using (var gzip = new GZipStream(bufferedStream, CompressionMode.Compress, leaveOpen: true))
            using (var stream = new CountingStream(gzip))
            {
                var binaryWriter = new BinaryWriter(stream);
                binaryWriter.Write(1);
                var bsonWriter = new BsonWriter(binaryWriter)
                {
                    DateTimeKindHandling = DateTimeKind.Unspecified
                };

                bsonWriter.WriteStartObject();

                bsonWriter.WritePropertyName(String.Empty);
                bsonWriter.WriteValue("ABCDEFG");

                bsonWriter.WriteEndObject();

                bsonWriter.Flush();
                binaryWriter.Flush();
                stream.Flush();
                bytesWritten = stream.NumberOfWrittenBytes;
            }
        }
Beispiel #2
0
        private void WriteToken(WebRequest httpWebRequest)
        {
            if (EnvironmentUtils.RunningOnPosix) // mono must set ContentLength before GetRequestStream (unlike .net)
            {
                httpWebRequest.ContentLength = postedToken.ToString().Length;
            }

            using (var stream = httpWebRequest.GetRequestStream())
            {
                using (var countingStream = new CountingStream(stream, l => NumberOfBytesWrittenCompressed = l))
                    using (var commpressedData = new GZipStream(countingStream, CompressionMode.Compress))
                        using (var countingStream2 = new CountingStream(commpressedData, l => NumberOfBytesWrittenUncompressed = l))
                        {
                            if (writeBson)
                            {
                                postedToken.WriteTo(new BsonWriter(countingStream2));
                            }
                            else
                            {
                                var streamWriter = new StreamWriter(countingStream2);
                                postedToken.WriteTo(new JsonTextWriter(streamWriter));
                                streamWriter.Flush();
                            }
                            commpressedData.Flush();
                            stream.Flush();
                        }
            }
        }
Beispiel #3
0
 public void Write(byte[] data)
 {
     postedData = data;
     using (var stream = WebRequest.GetRequestStream())
         using (var countingStream = new CountingStream(stream, l => NumberOfBytesWrittenCompressed = l))
             using (var cmp = new GZipStream(countingStream, CompressionMode.Compress))
                 using (var countingStream2 = new CountingStream(cmp, l => NumberOfBytesWrittenUncompressed = l))
                 {
                     countingStream2.Write(data, 0, data.Length);
                     cmp.Flush();
                     stream.Flush();
                 }
 }
Beispiel #4
0
 public void Write(Stream streamToWrite)
 {
     postedStream = streamToWrite;
     using (var stream = WebRequest.GetRequestStream())
         using (var countingStream = new CountingStream(stream, l => NumberOfBytesWrittenCompressed = l))
             using (var commpressedStream = new GZipStream(countingStream, CompressionMode.Compress))
                 using (var countingStream2 = new CountingStream(commpressedStream, l => NumberOfBytesWrittenUncompressed = l))
                 {
                     streamToWrite.CopyTo(countingStream2);
                     commpressedStream.Flush();
                     stream.Flush();
                 }
 }
Beispiel #5
0
        public void Write(Stream streamToWrite)
        {
            postedStream = streamToWrite;
            if (EnvironmentUtils.RunningOnPosix) // mono must set ContentLength before GetRequestStream (unlike .net)
            {
                WebRequest.ContentLength = streamToWrite.Length;
            }

            using (var stream = WebRequest.GetRequestStream())
                using (var countingStream = new CountingStream(stream, l => NumberOfBytesWrittenCompressed = l))
                    using (var commpressedStream = new GZipStream(countingStream, CompressionMode.Compress))
                        using (var countingStream2 = new CountingStream(commpressedStream, l => NumberOfBytesWrittenUncompressed = l))
                        {
                            streamToWrite.CopyTo(countingStream2);
                            commpressedStream.Flush();
                            stream.Flush();
                        }
        }
Beispiel #6
0
        public void Write(byte[] data)
        {
            postedData = data;
            if (EnvironmentUtils.RunningOnPosix) // mono must set ContentLength before GetRequestStream (unlike .net)
            {
                WebRequest.ContentLength = data.Length;
            }

            using (var stream = WebRequest.GetRequestStream())
                using (var countingStream = new CountingStream(stream, l => NumberOfBytesWrittenCompressed = l))
                    using (var cmp = new GZipStream(countingStream, CompressionMode.Compress))
                        using (var countingStream2 = new CountingStream(cmp, l => NumberOfBytesWrittenUncompressed = l))
                        {
                            countingStream2.Write(data, 0, data.Length);
                            cmp.Flush();
                            stream.Flush();
                        }
        }
Beispiel #7
0
 private void WriteToken(WebRequest httpWebRequest)
 {
     using (var stream = httpWebRequest.GetRequestStream())
         using (var countingStream = new CountingStream(stream, l => NumberOfBytesWrittenCompressed = l))
             using (var commpressedData = new GZipStream(countingStream, CompressionMode.Compress))
                 using (var countingStream2 = new CountingStream(commpressedData, l => NumberOfBytesWrittenUncompressed = l))
                 {
                     if (writeBson)
                     {
                         postedToken.WriteTo(new BsonWriter(countingStream2));
                     }
                     else
                     {
                         var streamWriter = new StreamWriter(countingStream2);
                         postedToken.WriteTo(new JsonTextWriter(streamWriter));
                         streamWriter.Flush();
                     }
                     commpressedData.Flush();
                     stream.Flush();
                 }
 }
        private RavenJObject ReadDocumentData(string normalizedKey, Slice sliceKey, Etag existingEtag, RavenJObject metadata, out int size)
        {
            try
            {
                size = -1;

                var existingCachedDocument = documentCacher.GetCachedDocument(normalizedKey, existingEtag);
                if (existingCachedDocument != null)
                {
                    size = existingCachedDocument.Size;
                    return existingCachedDocument.Document;
                }

                var documentReadResult = tableStorage.Documents.Read(Snapshot, sliceKey, writeBatch.Value);
                if (documentReadResult == null) //non existing document
                    return null;

                using (var stream = documentReadResult.Reader.AsStream())
                {
                    using (var decodedDocumentStream = documentCodecs.Aggregate(stream,
                            (current, codec) => codec.Value.Decode(normalizedKey, metadata, current)))
                    {
                        var streamToUse = decodedDocumentStream;
                        if (stream != decodedDocumentStream)
                            streamToUse = new CountingStream(decodedDocumentStream);

                        var documentData = decodedDocumentStream.ToJObject();

                        size = (int)Math.Max(stream.Position, streamToUse.Position);
                        documentCacher.SetCachedDocument(normalizedKey, existingEtag, documentData, metadata, size);

                        return documentData;
                    }
                }
            }
            catch (Exception e)
            { 
                InvalidDataException invalidDataException = null;
                try
                {
                    size = -1;
                    var documentReadResult = tableStorage.Documents.Read(Snapshot, sliceKey, writeBatch.Value);
                    if (documentReadResult == null) //non existing document
                        return null;

                    using (var stream = documentReadResult.Reader.AsStream())
                    {
                        using (var reader = new BinaryReader(stream))
                        {
                            if (reader.ReadUInt32() == DocumentCompression.CompressFileMagic)
                            {
                                invalidDataException = new InvalidDataException(string.Format("Document '{0}' is compressed, but the compression bundle is not enabled.\r\n" +
                                                                                              "You have to enable the compression bundle when dealing with compressed documents.", normalizedKey), e);
                            }
                        }
                    }

            
                }
                catch (Exception)
                {
                    // we are already in error handling mode, just ignore this
                }
                if(invalidDataException != null)
                    throw invalidDataException;

                throw new InvalidDataException("Failed to de-serialize a document: " + normalizedKey, e);
            }
        }
Beispiel #9
0
        public void WaitForIndexing()
        {
            while (true)
            {
                var response = httpClient.GetAsync(databaseUrl + "/stats").ResultUnwrap();

                if (response.IsSuccessStatusCode == false)
                    throw new InvalidOperationException(string.Format("STATS failed. Code: {0}.", response.StatusCode));

                using (var stream = response.GetResponseStreamWithHttpDecompression().ResultUnwrap())
                {
                    var countingStream = new CountingStream(stream);
                    var stats = (RavenJObject)RavenJToken.TryLoad(countingStream);

                    var staleIndexes = (RavenJArray)stats["StaleIndexes"];
                    if (staleIndexes.Length == 0)
                        return;

                    Thread.Sleep(1000);
                }
            }
        }
Beispiel #10
0
        public JsonDocument Get(string key)
        {
            var response = httpClient.GetAsync(databaseUrl + "/docs/" + Uri.EscapeUriString(key)).ResultUnwrap();

            if (response.IsSuccessStatusCode == false)
                throw new InvalidOperationException(string.Format("GET failed. Code: {0}.", response.StatusCode));

            using (var stream = response.GetResponseStreamWithHttpDecompression().ResultUnwrap())
            {
                var countingStream = new CountingStream(stream);
                var data = RavenJToken.TryLoad(countingStream);

                var docKey = Uri.UnescapeDataString(response.Headers.GetFirstValue(Constants.DocumentIdFieldName));

                response.Headers.Remove(Constants.DocumentIdFieldName);

                return new JsonDocument
                {
                    DataAsJson = (RavenJObject)data,
                    Metadata = response.Headers.FilterHeadersToObject(),
                    Key = docKey
                };
            }
        }
		private RavenJObject ReadDocumentData(string key, Etag existingEtag, RavenJObject metadata, out int size)
		{
			var loweredKey = CreateKey(key);

			size = -1;
			
			var existingCachedDocument = documentCacher.GetCachedDocument(loweredKey, existingEtag);
			if (existingCachedDocument != null)
			{
				size = existingCachedDocument.Size;
				return existingCachedDocument.Document;
			}

			var documentReadResult = tableStorage.Documents.Read(Snapshot, loweredKey, writeBatch.Value);
			if (documentReadResult == null) //non existing document
				return null;

			using (var stream = documentReadResult.Reader.AsStream())
			{
				using (var decodedDocumentStream = documentCodecs.Aggregate(stream,
						(current, codec) => codec.Value.Decode(loweredKey, metadata, current)))
				{
					var streamToUse = decodedDocumentStream;
					if(stream != decodedDocumentStream)
						streamToUse = new CountingStream(decodedDocumentStream);

					var documentData = decodedDocumentStream.ToJObject();

					size = (int)Math.Max(stream.Position, streamToUse.Position);

					documentCacher.SetCachedDocument(loweredKey, existingEtag, documentData, metadata, size);
					return documentData;	
				}
			}
		}
		public async virtual Task ImportData(Stream stream, SmugglerOptions options)
		{
			options = options ?? SmugglerOptions;
			if (options == null)
				throw new ArgumentNullException("options");

			await DetectServerSupportedFeatures();

			await EnsureDatabaseExists();
			Stream sizeStream;

			var sw = Stopwatch.StartNew();
			// Try to read the stream compressed, otherwise continue uncompressed.
			JsonTextReader jsonReader;
			try
			{
				sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
				var streamReader = new StreamReader(sizeStream);

				jsonReader = new JsonTextReader(streamReader);

				if (jsonReader.Read() == false)
					return;
			}
			catch (Exception e)
			{
			    if (e is InvalidDataException == false 
#if SILVERLIGHT
                    && e is ZlibException == false
#endif
                    )
			        throw;

				stream.Seek(0, SeekOrigin.Begin);

                sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));

				var streamReader = new StreamReader(stream);

				jsonReader = new JsonTextReader(streamReader);

				if (jsonReader.Read() == false)
					return;
			}

			if (jsonReader.TokenType != JsonToken.StartObject)
				throw new InvalidDataException("StartObject was expected");

			ShowProgress("Begin reading indexes");
			var indexCount = await ImportIndexes(jsonReader, options);
			ShowProgress(string.Format("Done with reading indexes, total: {0}", indexCount));

			ShowProgress("Begin reading documents");
			var documentCount = await ImportDocuments(jsonReader, options);
			ShowProgress(string.Format("Done with reading documents, total: {0}", documentCount));

			ShowProgress("Begin reading attachments");
			var attachmentCount = await ImportAttachments(jsonReader, options);
			ShowProgress(string.Format("Done with reading attachments, total: {0}", attachmentCount));

			ShowProgress("Begin reading transformers");
			var transformersCount = await ImportTransformers(jsonReader, options);
			ShowProgress(string.Format("Done with reading transformers, total: {0}", transformersCount));

			sw.Stop();

			ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", documentCount, attachmentCount, sw.ElapsedMilliseconds);
		}
Beispiel #13
0
		public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true)
		{
			EnsureDatabaseExists();
			Stream sizeStream;

			var sw = Stopwatch.StartNew();
			// Try to read the stream compressed, otherwise continue uncompressed.
			JsonTextReader jsonReader;
			try
			{
				sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
				var streamReader = new StreamReader(sizeStream);

				jsonReader = new JsonTextReader(streamReader);

				if (jsonReader.Read() == false)
					return;
			}
			catch (InvalidDataException)
			{
				sizeStream = stream;
				stream.Seek(0, SeekOrigin.Begin);

				var streamReader = new StreamReader(stream);

				jsonReader = new JsonTextReader(streamReader);

				if (jsonReader.Read() == false)
					return;
			}

			if (jsonReader.TokenType != JsonToken.StartObject)
				throw new InvalidDataException("StartObject was expected");

			// should read indexes now
			if (jsonReader.Read() == false)
				return;
			if (jsonReader.TokenType != JsonToken.PropertyName)
				throw new InvalidDataException("PropertyName was expected");
			if (Equals("Indexes", jsonReader.Value) == false)
				throw new InvalidDataException("Indexes property was expected");
			if (jsonReader.Read() == false)
				return;
			if (jsonReader.TokenType != JsonToken.StartArray)
				throw new InvalidDataException("StartArray was expected");

			while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
			{
				var index = RavenJToken.ReadFrom(jsonReader);
				if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes)
					continue;
				var indexName = index.Value<string>("name");
				if (indexName.StartsWith("Temp/"))
					continue;
				if (index.Value<RavenJObject>("definition").Value<bool>("IsCompiled"))
					continue; // can't import compiled indexes
				PutIndex(indexName, index);
			}

			// should read documents now
			if (jsonReader.Read() == false)
				return;
			if (jsonReader.TokenType != JsonToken.PropertyName)
				throw new InvalidDataException("PropertyName was expected");
			if (Equals("Docs", jsonReader.Value) == false)
				throw new InvalidDataException("Docs property was expected");
			if (jsonReader.Read() == false)
				return;
			if (jsonReader.TokenType != JsonToken.StartArray)
				throw new InvalidDataException("StartArray was expected");
			var batch = new List<RavenJObject>();
			int totalCount = 0;
			long lastFlushedAt = 0;
			int batchCount = 0;
			long sizeOnDisk = 0;
			while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
			{
				var before = sizeStream.Position;
				var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
				var size = sizeStream.Position - before;
				if (size > 1024 * 1024)
				{
					Console.WriteLine("{0:#,#.##;;0} kb - {1}",
						(double)size / 1024,
						document["@metadata"].Value<string>("@id"));
				}
				if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
					continue;
				if (options.MatchFilters(document) == false)
					continue;

				totalCount += 1;
				batch.Add(document);
				sizeOnDisk = (sizeStream.Position - lastFlushedAt);
				if (batch.Count >= smugglerOptions.BatchSize ||
					sizeOnDisk >= MaxSizeOfUncompressedSizeToSendToDatabase)
				{
					lastFlushedAt = sizeStream.Position;
					HandleBatch(options,batch, sizeOnDisk);
					sizeOnDisk = 0;
					if (++batchCount % 10 == 0)
					{
						OutputIndexingDistance();
					}
				}
			}
			HandleBatch(options, batch, sizeOnDisk);
			OutputIndexingDistance();

			var attachmentCount = 0;
			if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
				return;
			if (jsonReader.TokenType != JsonToken.PropertyName)
				throw new InvalidDataException("PropertyName was expected");
			if (Equals("Attachments", jsonReader.Value) == false)
				throw new InvalidDataException("Attachment property was expected");
			if (jsonReader.Read() == false)
				return;
			if (jsonReader.TokenType != JsonToken.StartArray)
				throw new InvalidDataException("StartArray was expected");
			while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
			{
				attachmentCount += 1;
				var item = RavenJToken.ReadFrom(jsonReader);
				if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments)
					continue;
				var attachmentExportInfo =
					new JsonSerializer
						{
							Converters = { new JsonToJsonConverter() }
						}.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item));
				ShowProgress("Importing attachment {0}", attachmentExportInfo.Key);

				PutAttachment(attachmentExportInfo);
			}
			ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
		}
		public async virtual Task ImportData(SmugglerImportOptions importOptions, Stream stream)
		{
			Operations.Configure(SmugglerOptions);
			Operations.Initialize(SmugglerOptions);
			await DetectServerSupportedFeatures(importOptions.To);

			Stream sizeStream;

			var sw = Stopwatch.StartNew();
			// Try to read the stream compressed, otherwise continue uncompressed.
			JsonTextReader jsonReader;
			try
			{
				stream.Position = 0;
				sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
				var streamReader = new StreamReader(sizeStream);

				jsonReader = new JsonTextReader(streamReader);

				if (jsonReader.Read() == false)
					return;
			}
			catch (Exception e)
			{
				if (e is InvalidDataException == false)
					throw;

				stream.Seek(0, SeekOrigin.Begin);

				sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));

				var streamReader = new StreamReader(sizeStream);

				jsonReader = new JsonTextReader(streamReader);

				if (jsonReader.Read() == false)
					return;
			}

			if (jsonReader.TokenType != JsonToken.StartObject)
				throw new InvalidDataException("StartObject was expected");

			var exportCounts = new Dictionary<string, int>();
			var exportSectionRegistar = new Dictionary<string, Func<int>>();

			SmugglerOptions.CancelToken.Token.ThrowIfCancellationRequested();

			exportSectionRegistar.Add("Indexes", () =>
			{
				Operations.ShowProgress("Begin reading indexes");
				var indexCount = ImportIndexes(jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading indexes, total: {0}", indexCount));
				return indexCount;
			});
			
			exportSectionRegistar.Add("Docs", () =>
			{
				Operations.ShowProgress("Begin reading documents");
				var documentCount = ImportDocuments(jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading documents, total: {0}", documentCount));
				return documentCount;
			});

			exportSectionRegistar.Add("Attachments", () =>
			{
				Operations.ShowProgress("Begin reading attachments");
				var attachmentCount = ImportAttachments(importOptions.To, jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading attachments, total: {0}", attachmentCount));
				return attachmentCount;
			});

			exportSectionRegistar.Add("Transformers", () =>
			{
				Operations.ShowProgress("Begin reading transformers");
				var transformersCount = ImportTransformers(jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading transformers, total: {0}", transformersCount));
				return transformersCount;
			});

			exportSectionRegistar.Add("DocsDeletions", () =>
			{
				Operations.ShowProgress("Begin reading deleted documents");
				var deletedDocumentsCount = ImportDeletedDocuments(jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading deleted documents, total: {0}", deletedDocumentsCount));
				return deletedDocumentsCount;
			});

			exportSectionRegistar.Add("AttachmentsDeletions", () =>
			{
				Operations.ShowProgress("Begin reading deleted attachments");
				var deletedAttachmentsCount = ImportDeletedAttachments(jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading deleted attachments, total: {0}", deletedAttachmentsCount));
				return deletedAttachmentsCount;
			});

			exportSectionRegistar.Add("Identities", () =>
			{
				Operations.ShowProgress("Begin reading identities");
				var identitiesCount = ImportIdentities(jsonReader).Result;
				Operations.ShowProgress(string.Format("Done with reading identities, total: {0}", identitiesCount));
				return identitiesCount;
			});

			exportSectionRegistar.Keys.ForEach(k => exportCounts[k] = 0);

			while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndObject)
			{
				SmugglerOptions.CancelToken.Token.ThrowIfCancellationRequested();

				if (jsonReader.TokenType != JsonToken.PropertyName)
					throw new InvalidDataException("PropertyName was expected");
				Func<int> currentAction;
				var currentSection = jsonReader.Value.ToString();
				if (exportSectionRegistar.TryGetValue(currentSection, out currentAction) == false)
				{
					throw new InvalidDataException("Unexpected property found: " + jsonReader.Value);
				}
				if (jsonReader.Read() == false)
				{
					exportCounts[currentSection] = 0;
					continue;
				}

				if (jsonReader.TokenType != JsonToken.StartArray)
					throw new InvalidDataException("StartArray was expected");

				exportCounts[currentSection] = currentAction();
			}

			sw.Stop();

			Operations.ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments, deleted {2:#,#;;0} documents and {3:#,#;;0} attachments in {4:#,#.###;;0} s", exportCounts["Docs"], exportCounts["Attachments"], exportCounts["DocsDeletions"], exportCounts["AttachmentsDeletions"], sw.ElapsedMilliseconds / 1000f);

			SmugglerOptions.CancelToken.Token.ThrowIfCancellationRequested();
		}