コード例 #1
0
        private IEnumerable <ICommandData> MigratePerson1ToPerson2(RavenJObject doc, RavenJObject metadata)
        {
            var name = doc.Value <string>("Name");

            if (!string.IsNullOrEmpty(name))
            {
                doc["FirstName"] = name.Split(' ')[0];
                doc["LastName"]  = name.Split(' ')[1];
            }
            doc.Remove("Name");

            metadata[Constants.RavenClrType] = "RavenMigrations.Tests.Person2, RavenMigrations.Tests";

            var foobaz = new FooBaz
            {
                Id  = 1,
                Bar = "loaded"
            };

            var foobazDoc = RavenJObject.FromObject(foobaz);
            var meta      = new RavenJObject();

            meta[Constants.RavenEntityName] = "FooBazs";
            var cmd = new PutCommandData
            {
                Document = foobazDoc,
                Key      = "foobazs/" + foobaz.Id,
                Metadata = meta
            };

            return(new[] { cmd });
        }
コード例 #2
0
ファイル: CommandDataFactory.cs プロジェクト: GorelH/ravendb
    public static ICommandData CreateCommand(RavenJObject jsonCommand, TransactionInformation transactionInformation)
		{
			var key = jsonCommand["Key"].Value<string>();
			switch (jsonCommand.Value<string>("Method"))
			{
				case "PUT":
					var putCommand = new PutCommandData
					{
						Key = key,
						Etag = GetEtagFromCommand(jsonCommand),
						Document = jsonCommand["Document"] as RavenJObject,
						Metadata = jsonCommand["Metadata"] as RavenJObject,
						TransactionInformation = transactionInformation
					};
			        ValidateMetadataKeys(putCommand.Metadata);
			        return putCommand;
				case "DELETE":
					return new DeleteCommandData
					{
						Key = key,
						Etag = GetEtagFromCommand(jsonCommand),
						TransactionInformation = transactionInformation
					};
				case "PATCH":
					return new PatchCommandData
					{
						Key = key,
						Etag = GetEtagFromCommand(jsonCommand),
						TransactionInformation = transactionInformation,
						Metadata = jsonCommand["Metadata"] as RavenJObject,
						Patches = jsonCommand
							.Value<RavenJArray>("Patches")
							.Cast<RavenJObject>()
							.Select(PatchRequest.FromJson)
							.ToArray(),
						PatchesIfMissing = jsonCommand["PatchesIfMissing"] == null ? null : jsonCommand
							.Value<RavenJArray>("PatchesIfMissing")
							.Cast<RavenJObject>()
							.Select(PatchRequest.FromJson)
							.ToArray(),
						SkipPatchIfEtagMismatch = jsonCommand.ContainsKey("SkipPatchIfEtagMismatch") && jsonCommand.Value<bool>("SkipPatchIfEtagMismatch")
					};
				case "EVAL":
					var debug = jsonCommand["DebugMode"].Value<bool>();
					return new ScriptedPatchCommandData
					{
						Key = key,
						Metadata = jsonCommand["Metadata"] as RavenJObject,
						Etag = GetEtagFromCommand(jsonCommand),
						TransactionInformation = transactionInformation,
						Patch = ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("Patch")),
						PatchIfMissing = jsonCommand["PatchIfMissing"] == null ? null : ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("PatchIfMissing")),
						DebugMode = debug
					};
				default:
					throw new ArgumentException("Batching only supports PUT, PATCH, EVAL and DELETE.");
			}
		}
コード例 #3
0
        public static PutCommandData ToPutCommandData(this RavenJObject itemToSave)
        {
            var output = new PutCommandData
            {
                Document = itemToSave,
                Metadata = itemToSave.Value <RavenJObject>("@metadata"),
                Key      = itemToSave.Value <RavenJObject>("@metadata").Value <string>("@id")
            };

            return(output);
        }
コード例 #4
0
        public void PutEntities(List <object> entities)
        {
            var commands = new List <PutCommandData>();

            foreach (var entity in entities)
            {
                var command = new PutCommandData();
                command.Document = RavenJObject.FromObject(entity);
                command.Key      = GenerateId(entity);
                command.Metadata = CreateMetadata(entity);

                commands.Add(command);
            }

            var response = httpClient
                           .PostAsync(databaseUrl + "/bulk_docs", new JsonContent(RavenJToken.FromObject(commands)))
                           .ResultUnwrap();

            if (response.IsSuccessStatusCode == false)
            {
                throw new InvalidOperationException(string.Format("BATCH failed. Code: {0}.", response.StatusCode));
            }
        }
コード例 #5
0
        public void Run(ISagaInfo sagaInfo)
        {
            PrintHelper.PrintFunctionCallDetails("PopulateSagaUniqueIdentityRecords", sagaInfo);
            var documentStore = new DocumentStore
            {
                Url             = sagaInfo.RavenUrl,
                DefaultDatabase = sagaInfo.DatabaseName
            };

            documentStore.Initialize();

            var       existingSagaUniqueIdentityUniqueValues = DocumentHelper.GetExistingSagaUniqueIdentityUniqueValues(documentStore, sagaInfo);
            var       commandsList   = new List <ICommandData>();
            var       dictionarytest = new Dictionary <string, int>();
            const int pageSize       = 100;

            for (var start = 0; ; start += pageSize)
            {
                var sagaDocuments = documentStore.DatabaseCommands.GetDocuments(start, pageSize);
                foreach (var document in sagaDocuments)
                {
                    if (document.Key.StartsWith(sagaInfo.SagaKeyPrefix) && DocumentHelper.HasServicebusUniqueValue(document))
                    {
                        var body        = document.DataAsJson;
                        var uniqueValue = body.Value <string>(sagaInfo.UniqueField);
                        if (existingSagaUniqueIdentityUniqueValues.Exists(x => x == uniqueValue))
                        {
                            PrintHelper.PrintMessage(string.Format("Skip uniquevalue {0} because it already exists", uniqueValue));
                            continue; // Already has SagaUniqueIdentity record
                        }
                        var sagaId = document.Key.Replace(sagaInfo.SagaKeyPrefix + "/", "");

                        var documentBody = new RavenJObject();
                        documentBody.Add("SagaId", sagaId);
                        documentBody.Add("UniqueValue", uniqueValue);
                        documentBody.Add("SagaDocId", document.Key);

                        var documentMetadata = new RavenJObject();
                        documentMetadata.Add("Content-Type", "application/json; charset=utf-8");
                        documentMetadata.Add("Raven-Entity-Name", "SagaUniqueIdentity");
                        documentMetadata.Add("Raven-Clr-Type",
                                             "NServiceBus.SagaPersisters.Raven.SagaUniqueIdentity, NServiceBus.Core");

                        ICommandData command = new PutCommandData
                        {
                            Key      = DocumentHelper.GenerateUniqueIdentityRecordId(uniqueValue, sagaInfo),
                            Document = documentBody,
                            Metadata = documentMetadata,
                        };

                        if (dictionarytest.ContainsKey(uniqueValue))
                        {
                            dictionarytest[uniqueValue] += 1;
                        }
                        else
                        {
                            dictionarytest[uniqueValue] = 1;
                        }

                        commandsList.Add(command);
                    }
                }

                if (sagaDocuments.Length < pageSize)
                {
                    break;
                }
            }
            PrintHelper.PrintMessage(string.Format("Attempt to insert {0} SagaUniqueIdentity documents", commandsList.Count));
            foreach (var item in dictionarytest.Keys)
            {
                if (dictionarytest[item] > 1)
                {
                    PrintHelper.PrintMessage("the following has duplicate " + item + " : " + dictionarytest[item]);
                }
            }
            var results = documentStore.AsyncDatabaseCommands.BatchAsync(commandsList.ToArray());

            PrintHelper.PrintResults("PopulateSagaUniqueIdentityRecords", results);
        }
コード例 #6
0
ファイル: GetAppsJob.cs プロジェクト: arielorn/Win8Apps
        private void GetApps2(int pages = -1, int apps = -1, Operation operation = Operation.Full)
        {
            Store.Conventions.MaxNumberOfRequestsPerSession = 820;

            const string sitemapIndex = "http://apps.microsoft.com/windows/sitemap_index.xml";
            var          prefixLength = "http://apps.microsoft.com/windows/".Length;

            XNamespace xmlns = XNamespace.Get("http://www.sitemaps.org/schemas/sitemap/0.9");

            XmlNameTable table   = new NameTable();
            var          manager = new XmlNamespaceManager(table);

            manager.AddNamespace("", xmlns.NamespaceName);

            IEnumerable <string> sitemapPages;

            using (var response = _httpClient.GetAsync(sitemapIndex, HttpCompletionOption.ResponseContentRead).Result)
            {
                if (response.StatusCode != HttpStatusCode.OK)
                {
                    return;
                }

                using (var stream = response.Content.ReadAsStreamAsync().Result)
                {
                    XDocument sitemap = XDocument.Load(stream);

                    var sitemapIndexElem = sitemap.Element(xmlns.GetName("sitemapindex"));
                    var sitemaps         = sitemapIndexElem.Elements(xmlns.GetName("sitemap"));

                    sitemapPages = sitemaps
                                   .Select(x => x.Element(xmlns.GetName("loc"))).Select(x => x.Value);
                }
            }

            var globalAppsToCultures = new ConcurrentDictionary <string, AppInfo2>();

            if (pages > 0)
            {
                sitemapPages = sitemapPages.Take(pages);
            }

            Parallel.ForEach(
                sitemapPages,
                new ParallelOptions {
                MaxDegreeOfParallelism = 2
            },
                sitemapPage =>
            {
                Console.WriteLine("Requesting: {0}", sitemapPage);
                using (var response = _httpClient.GetAsync(sitemapPage, HttpCompletionOption.ResponseContentRead).Result)
                {
                    if (response.StatusCode != HttpStatusCode.OK)
                    {
                        return;
                    }

                    Console.WriteLine("Received: {0}", sitemapPage);

                    using (var stream = response.Content.ReadAsStreamAsync().Result)
                    {
                        Console.WriteLine("Loading: {0}", sitemapPage);

                        XDocument sitemapPageContent = XDocument.Load(stream);

                        var urlsetElem = sitemapPageContent.Element(xmlns.GetName("urlset"));
                        var urlsElems  = urlsetElem.Elements(xmlns.GetName("url"));
                        var appsLinks  = urlsElems.Select(x => x.Element(xmlns.GetName("loc")))
                                         .Select(x => x.Value);


                        foreach (var appLink in appsLinks)
                        {
                            var appId   = appLink.Substring(appLink.LastIndexOf('/') + 1);
                            var culture = appLink.Substring(prefixLength,
                                                            appLink.IndexOf('/', prefixLength) -
                                                            prefixLength);

                            if (appId == "ROW")
                            {
                                continue;
                            }

                            globalAppsToCultures.AddOrUpdate(
                                appId,
                                new AppInfo2
                            {
                                Id        = appId,
                                Url       = appLink,
                                Countries = new List <string>(new[] { culture })
                            },
                                (c, ai) =>
                            {
                                ai.Countries.Add(culture);
                                return(ai);
                            }
                                );
                        }
                        Console.WriteLine("Total Found {0} applications", globalAppsToCultures.Count);
                    }
                }
            });

            AppIds = new HashSet <string>(globalAppsToCultures.Keys);

            if (LoadIdsOnly)
            {
                return;
            }

            var toProcess = globalAppsToCultures.AsEnumerable();

            if (apps > 0)
            {
                toProcess = toProcess.Take(apps);
            }

            var toProcessList = toProcess.ToList();

            var countdown = new CountdownEvent(toProcessList.Count);

            var commands = new List <ICommandData>();

            Parallel.ForEach(
                toProcessList,
                new ParallelOptions {
                MaxDegreeOfParallelism = 8
            },
                (kvp, pls, index) =>
            {
                using (var session = Store.OpenSession())
                {
                    try
                    {
                        Console.WriteLine("({0}/{1} - {2} remaining) Loading app {3}",
                                          index, globalAppsToCultures.Count, countdown.CurrentCount,
                                          kvp.Value.Id);

                        var existing = session.Load <App>("apps/" + kvp.Value.Id);
                        try
                        {
                            var app = GetAppDetails2(kvp.Value.Id, kvp.Value.Countries);
                            if (app == null)
                            {
                                return;
                            }

                            if (existing != null)
                            {
                                if ((app.Price != null && existing.Price != null &&
                                     app.Price.Amount == existing.Price.Amount) ||
                                    app.LastUpdated == existing.LastUpdated)
                                {
                                    return;
                                }
                                app.FriendlyUrl   = existing.FriendlyUrl;
                                app.PreviousPrice = existing.Price;
                            }

                            var putCommandData = new PutCommandData
                            {
                                Key      = app.Id,
                                Document = RavenJObject.FromObject(app),
                                Metadata = new RavenJObject
                                {
                                    { "Raven-Entity-Name", new RavenJValue("Apps") },
                                    {
                                        "Raven-Clr-Type",
                                        new RavenJValue("Win8Apps.Model.App, Win8Apps.Model")
                                    }
                                }
                            };

                            lock (commands)
                            {
                                commands.Add(putCommandData);
                            }
                        }
                        catch (Exception ex)
                        {
                        }
                    }
                    finally
                    {
                        countdown.Signal();
                    }
                }
            });

            countdown.Wait();

            if (commands.Count > 0)
            {
                const int batchSize = 512;
                int       handled   = 0;
                do
                {
                    var commandsToSave = commands
                                         .Skip(handled)
                                         .Take(batchSize)
                                         .ToList();

                    Store.DatabaseCommands.Batch(commandsToSave);
                    handled += commandsToSave.Count();

                    Console.WriteLine("Saving {0} {1}/{2} applications", commandsToSave.Count(), handled, commands.Count);
                    Thread.Sleep(5000);
                } while (handled < commands.Count);
            }
        }