/// <summary>
/// Initialize a HashPartitionResolver.
/// </summary>
/// <param name="partitionKeyPropertyName">The property name to be used as the partition Key.</param>
/// <param name="client">The DocumentDB client instance to use.</param>
/// <param name="database">The database to run the samples on.</param>
/// <param name="collectionNames">The names of collections used.</param>
/// <returns>The created HashPartitionResolver.</returns>
public static async Task <HashPartitionResolver> InitializeHashResolver(string partitionKeyPropertyName, DocumentClient client, Database database, string[] collectionNames)
{
// Set local to input.
string[] CollectionNames = collectionNames;
int numCollectionNames = CollectionNames.Length;
// Create array of DocumentCollections.
DocumentCollection[] collections = new DocumentCollection[numCollectionNames];
// Create string array of Self Links to Collections.
string[] selfLinks = new string[numCollectionNames];
//Create some collections to partition data.
for (int i = 0; i < numCollectionNames; i++)
{
collections[i] = await DocumentClientHelper.GetCollectionAsync(client, database, CollectionNames[i]);
selfLinks[i] = collections[i].SelfLink;
}
// Join Self Link Array into a comma separated string.
string selfLinkString = String.Join(", ", selfLinks);
//Initialize a partition resolver that users hashing, and register with DocumentClient.
//Uses User Id for PartitionKeyPropertyName, could also be TenantId, or any other variable.
HashPartitionResolver hashResolver = new HashPartitionResolver(partitionKeyPropertyName, new[] { selfLinkString });
client.PartitionResolvers[database.SelfLink] = hashResolver;
return(hashResolver);
}
/// <summary>
/// Initialize a HashPartitionResolver that uses a custom function to extract the partition key.
/// </summary>
/// <param name="partitionKeyExtractor">The partition key extractor function.</param>
/// <param name="client">The DocumentDB client instance to use.</param>
/// <param name="database">The database to run the samples on.</param>
/// <param name="collectionNames">The names of collections used.</param>
/// <returns>The created HashPartitionResolver.</returns>
public static async Task <HashPartitionResolver> InitializeCustomHashResolver(Func <object, string> partitionKeyExtractor, DocumentClient client, Database database, string[] collectionNames)
{
// Set local to input.
string[] CollectionNames = collectionNames;
int numCollectionNames = CollectionNames.Length;
// Create array of DocumentCollections.
DocumentCollection[] collections = new DocumentCollection[numCollectionNames];
// Create string array of Self Links to Collections.
string[] selfLinks = new string[numCollectionNames];
//Create some collections to partition data.
for (int i = 0; i < numCollectionNames; i++)
{
collections[i] = await DocumentClientHelper.GetCollectionAsync(client, database, CollectionNames[i]);
selfLinks[i] = collections[i].SelfLink;
}
// Join Self Link Array into a comma separated string.
string selfLinkString = String.Join(", ", selfLinks);
var hashResolver = new HashPartitionResolver(
partitionKeyExtractor,
new[] { selfLinkString });
client.PartitionResolvers[database.SelfLink] = hashResolver;
return(hashResolver);
}
/// <summary>
/// Initialize a HashPartitionResolver.
/// </summary>
/// <param name="partitionKeyPropertyName">The property name to be used as the partition Key.</param>
/// <param name="client">The DocumentDB client instance to use.</param>
/// <param name="database">The database to run the samples on.</param>
/// <param name="collectionNames">The names of collections used.</param>
/// <returns>The created HashPartitionResolver.</returns>
public static async Task<HashPartitionResolver> InitializeHashResolver(string partitionKeyPropertyName, DocumentClient client, Database database, string[] collectionNames)
{
// Set local to input.
string[] CollectionNames = collectionNames;
int numCollectionNames = CollectionNames.Length;
// Create array of DocumentCollections.
DocumentCollection[] collections = new DocumentCollection[numCollectionNames];
// Create string array of Self Links to Collections.
string[] selfLinks = new string[numCollectionNames];
//Create some collections to partition data.
for (int i = 0; i < numCollectionNames; i++)
{
collections[i] = await DocumentClientHelper.GetCollectionAsync(client, database, CollectionNames[i]);
selfLinks[i] = collections[i].SelfLink;
}
// Join Self Link Array into a comma separated string.
string selfLinkString = String.Join(", ", selfLinks);
//Initialize a partition resolver that users hashing, and register with DocumentClient.
//Uses User Id for PartitionKeyPropertyName, could also be TenantId, or any other variable.
HashPartitionResolver hashResolver = new HashPartitionResolver(partitionKeyPropertyName, new[] { selfLinkString });
client.PartitionResolvers[database.SelfLink] = hashResolver;
return hashResolver;
}
/// <summary>
/// Initializes the partition resolver to be used with Item repository.
/// </summary>
/// <param name="resolverService"></param>
public async void getResolver(int resolverService)
{
switch (resolverService)
{
case Hash:
HashPartitionResolver hashResolver = await PartitionInitializers.InitializeHashResolver("tenantId", Client, Database, collections);
break;
case ManagedHash:
ManagedHashPartitionResolver managedHashResolver = PartitionInitializers.InitializeManagedHashResolver(i => ((Item)i).tenantId, Client, Database, 3, null);
break;
case Spillover:
SpilloverPartitionResolver spilloverResolver = new SpilloverPartitionResolver(Client, Database);
Client.PartitionResolvers[Database.SelfLink] = spilloverResolver;
break;
case Range:
RangePartitionResolver <string> rangeResolver = await PartitionInitializers.InitializeRangeResolver("tenantId", Client, Database, collections);
break;
case Lookup:
LookupPartitionResolver <string> lookupResolver = await PartitionInitializers.InitializeLookupPartitionResolver("tenantId", Client, Database, collections);
break;
default:
goto case Hash;
}
}
public static IPartitionResolver CreateHashPartitionResolver(IReliableReadWriteDocumentClient client, Database db,
Func <object, string> partitionKeyExtractor, IEnumerable <string> partitionCollectionsSelfLinks)
{
//note that the partition key property must be present for all documents
HashPartitionResolver hashResolver = new HashPartitionResolver((Func <object, string>)partitionKeyExtractor, partitionCollectionsSelfLinks);
client.UnderlyingClient.PartitionResolvers[db.SelfLink] = hashResolver;
return(hashResolver);
}
public async Task <IHttpActionResult> HashPartitionResolver()
{
string[] collections = AppSettingsConfig.MainCollection.Split(',');
var database = await DocumentClientHelper.GetNewDatabaseAsync(_client, AppSettingsConfig.Db);
HashPartitionResolver hashResolver = await PartitionInitializers.InitializeHashResolver("UserId", _client, database, collections);
return(Ok());
}
/// <summary>
/// Initialize a HashPartitionResolver that uses a custom function to extract the partition key.
/// </summary>
/// <param name="database">The database to run the samples on.</param>
/// <returns>The created HashPartitionResolver.</returns>
private async Task <HashPartitionResolver> InitializeCustomHashResolver(Database database)
{
DocumentCollection collection1 = await DocumentClientHelper.GetCollectionAsync(this.client, database, "Collection.HashBucket0");
DocumentCollection collection2 = await DocumentClientHelper.GetCollectionAsync(this.client, database, "Collection.HashBucket1");
var hashResolver = new HashPartitionResolver(
u => ((UserProfile)u).UserId,
new[] { collection1.SelfLink, collection2.SelfLink });
this.client.PartitionResolvers[database.SelfLink] = hashResolver;
return(hashResolver);
}
/// <summary>
/// Initialize a HashPartitionResolver.
/// </summary>
/// <param name="database">The database to run the samples on.</param>
/// <returns>The created HashPartitionResolver.</returns>
private async Task <HashPartitionResolver> InitializeHashResolver(Database database)
{
// Create some collections to partition data.
DocumentCollection collection1 = await DocumentClientHelper.GetCollectionAsync(this.client, database, "Collection.HashBucket0");
DocumentCollection collection2 = await DocumentClientHelper.GetCollectionAsync(this.client, database, "Collection.HashBucket1");
// Initialize a partition resolver that users hashing, and register with DocumentClient.
HashPartitionResolver hashResolver = new HashPartitionResolver("UserId", new[] { collection1.SelfLink, collection2.SelfLink });
this.client.PartitionResolvers[database.SelfLink] = hashResolver;
return(hashResolver);
}
/// <summary>
/// Illustrate how to load and save the serializer state. Uses HashPartitionResolver as example.
/// </summary>
private void RunSerializeDeserializeSample(HashPartitionResolver hashResolver)
{
// Store the partition resolver's configuration as JSON.
string resolverStateAsString = JsonConvert.SerializeObject(hashResolver);
// Read properties and extract individual values to clone the resolver.
// ISSUE: fix deserialization of IHashGenerator
dynamic parsedResolver = JsonConvert.DeserializeObject(resolverStateAsString);
string partitionKeyPropertyName = parsedResolver.PartitionKeyPropertyName;
JArray collections = parsedResolver.CollectionLinks;
List <string> collectionLinks = collections.Select(c => (string)c).ToList();
HashPartitionResolver clonedHashResolver = new HashPartitionResolver(partitionKeyPropertyName, collectionLinks);
}
private static async Task WriteCollectionSizes(HashPartitionResolver resolver)
{
foreach (var collectionLink in resolver.CollectionLinks)
{
var collection = await client.ReadDocumentCollectionAsync(collectionLink);
var feed = await client.ReadDocumentFeedAsync(collectionLink, new FeedOptions
{
MaxItemCount = 1000
});
Console.WriteLine("Collection '{0}', documents: {1}", collection.Resource.Id, feed.Count);
}
}
/// <summary>
/// Initializes the partition resolver to be used with Item repository.
/// </summary>
/// <param name="resolverService"></param>
public async void getResolver(int resolverService)
{
switch (resolverService)
{
case Hash:
HashPartitionResolver hashResolver = await PartitionInitializers.InitializeHashResolver("docType", Client, Database, collections);
break;
case ManagedHash:
ManagedHashPartitionResolver managedHashResolver = PartitionInitializers.InitializeManagedHashResolver(i => ((ItemBase)i).docType, Client, Database, 3, null);
break;
default:
goto case Hash;
}
}
/// <summary>
/// Initialize a HashPartitionResolver.
/// </summary>
/// <param name="database">The database to run the samples on.</param>
/// <returns>The created HashPartitionResolver.</returns>
private async Task<HashPartitionResolver> InitializeHashResolver(Database database)
{
// Create some collections to partition data.
DocumentCollection collection1 = await DocumentClientHelper.GetOrCreateCollectionAsync(this.client, database.Id, "Collection.HashBucket0");
DocumentCollection collection2 = await DocumentClientHelper.GetOrCreateCollectionAsync(this.client, database.Id, "Collection.HashBucket1");
// Initialize a partition resolver that users hashing, and register with DocumentClient.
HashPartitionResolver hashResolver = new HashPartitionResolver("UserId", new[] { collection1.SelfLink, collection2.SelfLink });
this.client.PartitionResolvers[database.SelfLink] = hashResolver;
return hashResolver;
}
public ItemRepository(DocumentClient client, string databaseId) : base(client, databaseId)
{
this.partitionResolver = new HashPartitionResolver(base.database.SelfLink);
this.partitionResolver.Initialize(client);
}
static int Main(string[] args)
{
Options options = new Options();
if (!CommandLine.Parser.Default.ParseArguments(args, options))
{
Console.WriteLine("Invalid arguments");
return 1;
}
using (DocumentClient client = new DocumentClient(
new Uri(options.Endpoint),
options.AuthKey,
new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }))
{
Database database = client.CreateDatabaseQuery().Where(d => d.Id == options.Database).AsEnumerable().FirstOrDefault();
if (database == null)
{
Console.WriteLine("Cannot find database " + options.Database);
return 2;
}
List<DocumentCollection> collections = client.ReadDocumentCollectionFeedAsync(database.SelfLink).Result.ToList();
int minimumRequiredCollections = Math.Max(options.NewCollections, options.CurrentCollections);
if (collections.Count < minimumRequiredCollections)
{
Console.WriteLine("At least {0} collections must be pre-created", minimumRequiredCollections);
return 3;
}
Console.WriteLine("Current distribution of documents across collections:");
LogDocumentCountsPerCollection(client, database).Wait();
Console.WriteLine();
HashPartitionResolver currentPartitionResolver = new HashPartitionResolver(options.PartitionKeyName, collections.Take(options.CurrentCollections).Select(c => c.SelfLink));
HashPartitionResolver nextPartitionResolver = new HashPartitionResolver(options.PartitionKeyName, collections.Take(options.NewCollections).Select(c => c.SelfLink));
int numberOfMovedDocuments = 0;
Parallel.ForEach(currentPartitionResolver.CollectionLinks, collectionLink =>
{
ResourceFeedReader<Document> feedReader = client.CreateDocumentFeedReader(collectionLink, new FeedOptions { MaxItemCount = -1 });
while (feedReader.HasMoreResults)
{
foreach (Document document in DocumentClientHelper.ExecuteWithRetryAsync<FeedResponse<Document>>(() => feedReader.ExecuteNextAsync()).Result)
{
object partitionKey = nextPartitionResolver.GetPartitionKey(document);
string newCollectionLink = nextPartitionResolver.ResolveForCreate(partitionKey);
if (newCollectionLink != collectionLink)
{
int count = Interlocked.Increment(ref numberOfMovedDocuments);
DocumentClientHelper.ExecuteWithRetryAsync(() => client.DeleteDocumentAsync(document.SelfLink)).Wait();
DocumentClientHelper.ExecuteWithRetryAsync(() => client.CreateDocumentAsync(newCollectionLink, document)).Wait();
if (count % 100 == 0)
{
Console.WriteLine("Moved {0} documents between partitions", numberOfMovedDocuments);
}
}
}
}
});
Console.WriteLine();
Console.WriteLine("Moved {0} documents between partitions.", numberOfMovedDocuments);
Console.WriteLine();
Console.WriteLine("Current distribution of documents across collections:");
LogDocumentCountsPerCollection(client, database).Wait();
Console.WriteLine();
}
return 0;
}
/// <summary>
/// Initialize a HashPartitionResolver that uses a custom function to extract the partition key.
/// </summary>
/// <param name="database">The database to run the samples on.</param>
/// <returns>The created HashPartitionResolver.</returns>
private async Task<HashPartitionResolver> InitializeCustomHashResolver(Database database)
{
DocumentCollection collection1 = await DocumentClientHelper.GetOrCreateCollectionAsync(this.client, database.Id, "Collection.HashBucket0");
DocumentCollection collection2 = await DocumentClientHelper.GetOrCreateCollectionAsync(this.client, database.Id, "Collection.HashBucket1");
var hashResolver = new HashPartitionResolver(
u => ((UserProfile)u).UserId,
new[] { collection1.SelfLink, collection2.SelfLink });
this.client.PartitionResolvers[database.SelfLink] = hashResolver;
return hashResolver;
}
/// <summary>
/// Run samples for Database create, read, update and delete.
/// </summary>
/// <returns>a Task object.</returns>
private async Task RunAsync()
{
// Let's see how a to use a HashPartitionResolver.
Console.WriteLine("Running samples with the default hash partition resolver ...");
Database database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
HashPartitionResolver hashResolver = await this.InitializeHashResolver(database);
await this.RunCrudAndQuerySample(database, hashResolver);
// Let's see how to use a RangePartitionResolver.
Console.WriteLine("Running samples with the default range partition resolver ...");
database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
RangePartitionResolver <string> rangeResolver = await this.InitializeRangeResolver(database);
await this.RunCrudAndQuerySample(database, rangeResolver);
// Now let's take a look at an example of how to derive from one of the supported partition resolvers.
// Here we implement a generic hash resolver that takes an arbirary lambda to extract partition keys.
database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
HashPartitionResolver customHashResolver = await this.InitializeCustomHashResolver(database);
await this.RunCrudAndQuerySample(database, customHashResolver);
// Let's take a look at a example of a LookupPartitionResolver, i.e., use a simple lookup table.
Console.WriteLine("Running samples with a lookup partition resolver ...");
database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
LookupPartitionResolver <string> lookupResolver = await this.InitializeLookupPartitionResolver(database);
await this.RunCrudAndQuerySample(database, lookupResolver);
// Now, a "managed" HashPartitionResolver that creates collections, while cloning their attributes like
// IndexingPolicy and OfferType.
Console.WriteLine("Running samples with a custom hash partition resolver that automates creating collections ...");
database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
ManagedHashPartitionResolver managedHashResolver = this.InitializeManagedHashResolver(database);
await this.RunCrudAndQuerySample(database, managedHashResolver);
// Now, a PartitionResolver that starts with one collection and spills over to new ones as the collections
// get filled up.
Console.WriteLine("Running samples with a custom \"spillover\" partition resolver");
database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
SpilloverPartitionResolver spilloverResolver = new SpilloverPartitionResolver(this.client, database);
this.client.PartitionResolvers[database.SelfLink] = spilloverResolver;
await this.RunCrudAndQuerySample(database, spilloverResolver);
// Now let's see how to persist the settings for a PartitionResolver, and how to bootstrap from those settings.
this.RunSerializeDeserializeSample(hashResolver);
// Now let's take a look at how to add and remove partitions to a HashPartitionResolver.
database = await DocumentClientHelper.GetNewDatabaseAsync(this.client, DatabaseId);
await this.RepartitionDataSample(database);
}
/// <summary>
/// Initialize a HashPartitionResolver that uses a custom function to extract the partition key.
/// </summary>
/// <param name="partitionKeyExtractor">The partition key extractor function.</param>
/// <param name="client">The DocumentDB client instance to use.</param>
/// <param name="database">The database to run the samples on.</param>
/// <param name="collectionNames">The names of collections used.</param>
/// <returns>The created HashPartitionResolver.</returns>
public static async Task<HashPartitionResolver> InitializeCustomHashResolver(Func<object, string> partitionKeyExtractor, DocumentClient client, Database database, string[] collectionNames)
{
// Set local to input.
string[] CollectionNames = collectionNames;
int numCollectionNames = CollectionNames.Length;
// Create array of DocumentCollections.
DocumentCollection[] collections = new DocumentCollection[numCollectionNames];
// Create string array of Self Links to Collections.
string[] selfLinks = new string[numCollectionNames];
//Create some collections to partition data.
for (int i = 0; i < numCollectionNames; i++)
{
collections[i] = await DocumentClientHelper.GetCollectionAsync(client, database, CollectionNames[i]);
selfLinks[i] = collections[i].SelfLink;
}
// Join Self Link Array into a comma separated string.
string selfLinkString = String.Join(", ", selfLinks);
var hashResolver = new HashPartitionResolver(
partitionKeyExtractor,
new[] { selfLinkString });
client.PartitionResolvers[database.SelfLink] = hashResolver;
return hashResolver;
}
/// <summary>
/// Illustrate how to load and save the serializer state. Uses HashPartitionResolver as example.
/// </summary>
private void RunSerializeDeserializeSample(HashPartitionResolver hashResolver)
{
// Store the partition resolver's configuration as JSON.
string resolverStateAsString = JsonConvert.SerializeObject(hashResolver);
// Read properties and extract individual values to clone the resolver.
// ISSUE: fix deserialization of IHashGenerator
dynamic parsedResolver = JsonConvert.DeserializeObject(resolverStateAsString);
string partitionKeyPropertyName = parsedResolver.PartitionKeyPropertyName;
JArray collections = parsedResolver.CollectionLinks;
List<string> collectionLinks = collections.Select(c => (string)c).ToList();
HashPartitionResolver clonedHashResolver = new HashPartitionResolver(partitionKeyPropertyName, collectionLinks);
}
static int Main(string[] args)
{
Options options = new Options();
if (!CommandLine.Parser.Default.ParseArguments(args, options))
{
Console.WriteLine("Invalid arguments");
return(1);
}
using (DocumentClient client = new DocumentClient(
new Uri(options.Endpoint),
options.AuthKey,
new ConnectionPolicy {
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp
}))
{
Database database = client.CreateDatabaseQuery().Where(d => d.Id == options.Database).AsEnumerable().FirstOrDefault();
if (database == null)
{
Console.WriteLine("Cannot find database " + options.Database);
return(2);
}
List <DocumentCollection> collections = client.ReadDocumentCollectionFeedAsync(database.SelfLink).Result.ToList();
int minimumRequiredCollections = Math.Max(options.NewCollections, options.CurrentCollections);
if (collections.Count < minimumRequiredCollections)
{
Console.WriteLine("At least {0} collections must be pre-created", minimumRequiredCollections);
return(3);
}
Console.WriteLine("Current distribution of documents across collections:");
LogDocumentCountsPerCollection(client, database).Wait();
Console.WriteLine();
HashPartitionResolver currentPartitionResolver = new HashPartitionResolver(options.PartitionKeyName, collections.Take(options.CurrentCollections).Select(c => c.SelfLink));
HashPartitionResolver nextPartitionResolver = new HashPartitionResolver(options.PartitionKeyName, collections.Take(options.NewCollections).Select(c => c.SelfLink));
int numberOfMovedDocuments = 0;
Parallel.ForEach(currentPartitionResolver.CollectionLinks, collectionLink =>
{
ResourceFeedReader <Document> feedReader = client.CreateDocumentFeedReader(collectionLink, new FeedOptions {
MaxItemCount = -1
});
while (feedReader.HasMoreResults)
{
foreach (Document document in DocumentClientHelper.ExecuteWithRetryAsync <FeedResponse <Document> >(() => feedReader.ExecuteNextAsync()).Result)
{
object partitionKey = nextPartitionResolver.GetPartitionKey(document);
string newCollectionLink = nextPartitionResolver.ResolveForCreate(partitionKey);
if (newCollectionLink != collectionLink)
{
int count = Interlocked.Increment(ref numberOfMovedDocuments);
DocumentClientHelper.ExecuteWithRetryAsync(() => client.DeleteDocumentAsync(document.SelfLink)).Wait();
DocumentClientHelper.ExecuteWithRetryAsync(() => client.CreateDocumentAsync(newCollectionLink, document)).Wait();
if (count % 100 == 0)
{
Console.WriteLine("Moved {0} documents between partitions", numberOfMovedDocuments);
}
}
}
}
});
Console.WriteLine();
Console.WriteLine("Moved {0} documents between partitions.", numberOfMovedDocuments);
Console.WriteLine();
Console.WriteLine("Current distribution of documents across collections:");
LogDocumentCountsPerCollection(client, database).Wait();
Console.WriteLine();
}
return(0);
}
