private IEnumerable <ICommandData> MigratePerson1ToPerson2(RavenJObject doc, RavenJObject metadata)
{
var name = doc.Value <string>("Name");
if (!string.IsNullOrEmpty(name))
{
doc["FirstName"] = name.Split(' ')[0];
doc["LastName"] = name.Split(' ')[1];
}
doc.Remove("Name");
metadata[Constants.RavenClrType] = "RavenMigrations.Tests.Person2, RavenMigrations.Tests";
var foobaz = new FooBaz
{
Id = 1,
Bar = "loaded"
};
var foobazDoc = RavenJObject.FromObject(foobaz);
var meta = new RavenJObject();
meta[Constants.RavenEntityName] = "FooBazs";
var cmd = new PutCommandData
{
Document = foobazDoc,
Key = "foobazs/" + foobaz.Id,
Metadata = meta
};
return(new[] { cmd });
}
public static ICommandData CreateCommand(RavenJObject jsonCommand, TransactionInformation transactionInformation)
{
var key = jsonCommand["Key"].Value<string>();
switch (jsonCommand.Value<string>("Method"))
{
case "PUT":
var putCommand = new PutCommandData
{
Key = key,
Etag = GetEtagFromCommand(jsonCommand),
Document = jsonCommand["Document"] as RavenJObject,
Metadata = jsonCommand["Metadata"] as RavenJObject,
TransactionInformation = transactionInformation
};
ValidateMetadataKeys(putCommand.Metadata);
return putCommand;
case "DELETE":
return new DeleteCommandData
{
Key = key,
Etag = GetEtagFromCommand(jsonCommand),
TransactionInformation = transactionInformation
};
case "PATCH":
return new PatchCommandData
{
Key = key,
Etag = GetEtagFromCommand(jsonCommand),
TransactionInformation = transactionInformation,
Metadata = jsonCommand["Metadata"] as RavenJObject,
Patches = jsonCommand
.Value<RavenJArray>("Patches")
.Cast<RavenJObject>()
.Select(PatchRequest.FromJson)
.ToArray(),
PatchesIfMissing = jsonCommand["PatchesIfMissing"] == null ? null : jsonCommand
.Value<RavenJArray>("PatchesIfMissing")
.Cast<RavenJObject>()
.Select(PatchRequest.FromJson)
.ToArray(),
SkipPatchIfEtagMismatch = jsonCommand.ContainsKey("SkipPatchIfEtagMismatch") && jsonCommand.Value<bool>("SkipPatchIfEtagMismatch")
};
case "EVAL":
var debug = jsonCommand["DebugMode"].Value<bool>();
return new ScriptedPatchCommandData
{
Key = key,
Metadata = jsonCommand["Metadata"] as RavenJObject,
Etag = GetEtagFromCommand(jsonCommand),
TransactionInformation = transactionInformation,
Patch = ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("Patch")),
PatchIfMissing = jsonCommand["PatchIfMissing"] == null ? null : ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("PatchIfMissing")),
DebugMode = debug
};
default:
throw new ArgumentException("Batching only supports PUT, PATCH, EVAL and DELETE.");
}
}
public static PutCommandData ToPutCommandData(this RavenJObject itemToSave)
{
var output = new PutCommandData
{
Document = itemToSave,
Metadata = itemToSave.Value <RavenJObject>("@metadata"),
Key = itemToSave.Value <RavenJObject>("@metadata").Value <string>("@id")
};
return(output);
}
public void PutEntities(List <object> entities)
{
var commands = new List <PutCommandData>();
foreach (var entity in entities)
{
var command = new PutCommandData();
command.Document = RavenJObject.FromObject(entity);
command.Key = GenerateId(entity);
command.Metadata = CreateMetadata(entity);
commands.Add(command);
}
var response = httpClient
.PostAsync(databaseUrl + "/bulk_docs", new JsonContent(RavenJToken.FromObject(commands)))
.ResultUnwrap();
if (response.IsSuccessStatusCode == false)
{
throw new InvalidOperationException(string.Format("BATCH failed. Code: {0}.", response.StatusCode));
}
}
public void Run(ISagaInfo sagaInfo)
{
PrintHelper.PrintFunctionCallDetails("PopulateSagaUniqueIdentityRecords", sagaInfo);
var documentStore = new DocumentStore
{
Url = sagaInfo.RavenUrl,
DefaultDatabase = sagaInfo.DatabaseName
};
documentStore.Initialize();
var existingSagaUniqueIdentityUniqueValues = DocumentHelper.GetExistingSagaUniqueIdentityUniqueValues(documentStore, sagaInfo);
var commandsList = new List <ICommandData>();
var dictionarytest = new Dictionary <string, int>();
const int pageSize = 100;
for (var start = 0; ; start += pageSize)
{
var sagaDocuments = documentStore.DatabaseCommands.GetDocuments(start, pageSize);
foreach (var document in sagaDocuments)
{
if (document.Key.StartsWith(sagaInfo.SagaKeyPrefix) && DocumentHelper.HasServicebusUniqueValue(document))
{
var body = document.DataAsJson;
var uniqueValue = body.Value <string>(sagaInfo.UniqueField);
if (existingSagaUniqueIdentityUniqueValues.Exists(x => x == uniqueValue))
{
PrintHelper.PrintMessage(string.Format("Skip uniquevalue {0} because it already exists", uniqueValue));
continue; // Already has SagaUniqueIdentity record
}
var sagaId = document.Key.Replace(sagaInfo.SagaKeyPrefix + "/", "");
var documentBody = new RavenJObject();
documentBody.Add("SagaId", sagaId);
documentBody.Add("UniqueValue", uniqueValue);
documentBody.Add("SagaDocId", document.Key);
var documentMetadata = new RavenJObject();
documentMetadata.Add("Content-Type", "application/json; charset=utf-8");
documentMetadata.Add("Raven-Entity-Name", "SagaUniqueIdentity");
documentMetadata.Add("Raven-Clr-Type",
"NServiceBus.SagaPersisters.Raven.SagaUniqueIdentity, NServiceBus.Core");
ICommandData command = new PutCommandData
{
Key = DocumentHelper.GenerateUniqueIdentityRecordId(uniqueValue, sagaInfo),
Document = documentBody,
Metadata = documentMetadata,
};
if (dictionarytest.ContainsKey(uniqueValue))
{
dictionarytest[uniqueValue] += 1;
}
else
{
dictionarytest[uniqueValue] = 1;
}
commandsList.Add(command);
}
}
if (sagaDocuments.Length < pageSize)
{
break;
}
}
PrintHelper.PrintMessage(string.Format("Attempt to insert {0} SagaUniqueIdentity documents", commandsList.Count));
foreach (var item in dictionarytest.Keys)
{
if (dictionarytest[item] > 1)
{
PrintHelper.PrintMessage("the following has duplicate " + item + " : " + dictionarytest[item]);
}
}
var results = documentStore.AsyncDatabaseCommands.BatchAsync(commandsList.ToArray());
PrintHelper.PrintResults("PopulateSagaUniqueIdentityRecords", results);
}
private void GetApps2(int pages = -1, int apps = -1, Operation operation = Operation.Full)
{
Store.Conventions.MaxNumberOfRequestsPerSession = 820;
const string sitemapIndex = "http://apps.microsoft.com/windows/sitemap_index.xml";
var prefixLength = "http://apps.microsoft.com/windows/".Length;
XNamespace xmlns = XNamespace.Get("http://www.sitemaps.org/schemas/sitemap/0.9");
XmlNameTable table = new NameTable();
var manager = new XmlNamespaceManager(table);
manager.AddNamespace("", xmlns.NamespaceName);
IEnumerable <string> sitemapPages;
using (var response = _httpClient.GetAsync(sitemapIndex, HttpCompletionOption.ResponseContentRead).Result)
{
if (response.StatusCode != HttpStatusCode.OK)
{
return;
}
using (var stream = response.Content.ReadAsStreamAsync().Result)
{
XDocument sitemap = XDocument.Load(stream);
var sitemapIndexElem = sitemap.Element(xmlns.GetName("sitemapindex"));
var sitemaps = sitemapIndexElem.Elements(xmlns.GetName("sitemap"));
sitemapPages = sitemaps
.Select(x => x.Element(xmlns.GetName("loc"))).Select(x => x.Value);
}
}
var globalAppsToCultures = new ConcurrentDictionary <string, AppInfo2>();
if (pages > 0)
{
sitemapPages = sitemapPages.Take(pages);
}
Parallel.ForEach(
sitemapPages,
new ParallelOptions {
MaxDegreeOfParallelism = 2
},
sitemapPage =>
{
Console.WriteLine("Requesting: {0}", sitemapPage);
using (var response = _httpClient.GetAsync(sitemapPage, HttpCompletionOption.ResponseContentRead).Result)
{
if (response.StatusCode != HttpStatusCode.OK)
{
return;
}
Console.WriteLine("Received: {0}", sitemapPage);
using (var stream = response.Content.ReadAsStreamAsync().Result)
{
Console.WriteLine("Loading: {0}", sitemapPage);
XDocument sitemapPageContent = XDocument.Load(stream);
var urlsetElem = sitemapPageContent.Element(xmlns.GetName("urlset"));
var urlsElems = urlsetElem.Elements(xmlns.GetName("url"));
var appsLinks = urlsElems.Select(x => x.Element(xmlns.GetName("loc")))
.Select(x => x.Value);
foreach (var appLink in appsLinks)
{
var appId = appLink.Substring(appLink.LastIndexOf('/') + 1);
var culture = appLink.Substring(prefixLength,
appLink.IndexOf('/', prefixLength) -
prefixLength);
if (appId == "ROW")
{
continue;
}
globalAppsToCultures.AddOrUpdate(
appId,
new AppInfo2
{
Id = appId,
Url = appLink,
Countries = new List <string>(new[] { culture })
},
(c, ai) =>
{
ai.Countries.Add(culture);
return(ai);
}
);
}
Console.WriteLine("Total Found {0} applications", globalAppsToCultures.Count);
}
}
});
AppIds = new HashSet <string>(globalAppsToCultures.Keys);
if (LoadIdsOnly)
{
return;
}
var toProcess = globalAppsToCultures.AsEnumerable();
if (apps > 0)
{
toProcess = toProcess.Take(apps);
}
var toProcessList = toProcess.ToList();
var countdown = new CountdownEvent(toProcessList.Count);
var commands = new List <ICommandData>();
Parallel.ForEach(
toProcessList,
new ParallelOptions {
MaxDegreeOfParallelism = 8
},
(kvp, pls, index) =>
{
using (var session = Store.OpenSession())
{
try
{
Console.WriteLine("({0}/{1} - {2} remaining) Loading app {3}",
index, globalAppsToCultures.Count, countdown.CurrentCount,
kvp.Value.Id);
var existing = session.Load <App>("apps/" + kvp.Value.Id);
try
{
var app = GetAppDetails2(kvp.Value.Id, kvp.Value.Countries);
if (app == null)
{
return;
}
if (existing != null)
{
if ((app.Price != null && existing.Price != null &&
app.Price.Amount == existing.Price.Amount) ||
app.LastUpdated == existing.LastUpdated)
{
return;
}
app.FriendlyUrl = existing.FriendlyUrl;
app.PreviousPrice = existing.Price;
}
var putCommandData = new PutCommandData
{
Key = app.Id,
Document = RavenJObject.FromObject(app),
Metadata = new RavenJObject
{
{ "Raven-Entity-Name", new RavenJValue("Apps") },
{
"Raven-Clr-Type",
new RavenJValue("Win8Apps.Model.App, Win8Apps.Model")
}
}
};
lock (commands)
{
commands.Add(putCommandData);
}
}
catch (Exception ex)
{
}
}
finally
{
countdown.Signal();
}
}
});
countdown.Wait();
if (commands.Count > 0)
{
const int batchSize = 512;
int handled = 0;
do
{
var commandsToSave = commands
.Skip(handled)
.Take(batchSize)
.ToList();
Store.DatabaseCommands.Batch(commandsToSave);
handled += commandsToSave.Count();
Console.WriteLine("Saving {0} {1}/{2} applications", commandsToSave.Count(), handled, commands.Count);
Thread.Sleep(5000);
} while (handled < commands.Count);
}
}
