public async Task <IEnumerable <Document> > ArchiveAllCourses(ILogger log, IEnumerable <Document> documents)
{
Throw.IfNull(log, nameof(log));
Throw.IfNullOrEmpty(documents, nameof(documents));
List <Document> responseList = new List <Document>();
if (documents.Any())
{
using (var client = _cosmosDbHelper.GetClient())
{
Uri uri = UriFactory.CreateDocumentCollectionUri(_dbSettings.DatabaseId, _settings.CoursesCollectionId);
FeedOptions options = new FeedOptions {
EnableCrossPartitionQuery = true, MaxItemCount = -1
};
foreach (var doc in documents)
{
int UKPRN = doc.GetPropertyValue <int?>("UnitedKingdomProviderReferenceNumber") ?? 0;
if (UKPRN > 10000000)
{
log.LogInformation($"Processing document with UKPRN {UKPRN}");
IEnumerable <Course> courseDocs = client.CreateDocumentQuery <Course>(uri, options)
.Where(x => x.ProviderUKPRN == UKPRN);
foreach (Course courseDoc in courseDocs)
{
log.LogInformation($"Archiving course with id {courseDoc.id}");
Uri docUri = UriFactory.CreateDocumentUri(_dbSettings.DatabaseId, _settings.CoursesCollectionId, courseDoc.id.ToString());
//var result = await client.DeleteDocumentAsync(docUri, new RequestOptions() { PartitionKey = new PartitionKey(doc.ProviderUKPRN) });
Document d = client.ReadDocumentAsync(docUri, new RequestOptions()
{
PartitionKey = new PartitionKey(UKPRN)
})
?.Result
?.Resource;
if (d == null)
{
log.LogInformation($"** Course with id {courseDoc.id} and Title {courseDoc.QualificationCourseTitle} wasn't archived");
}
else
{
d.SetPropertyValue("CourseStatus", (int)RecordStatus.Archived);
//result = await client.UpsertDocumentAsync(docUri, result.Resource);
d = await _cosmosDbHelper.UpdateDocumentAsync(client, _settings.CoursesCollectionId, d);
responseList.Add(d);
log.LogInformation($" archived successfully");
}
}
}
}
}
}
return(responseList);
}
public async Task Update(IReadOnlyList <Document> documents)
{
foreach (var doc in documents)
{
var newEntity = JsonConvert.DeserializeObject <Entity>(doc.ToString());
Uri databaseLink = UriFactory.CreateDatabaseUri(databaseName);
Uri collectionLink = UriFactory.CreateDocumentCollectionUri(databaseName, collectionName);
// Update player score
Uri docDocumentUri = UriFactory.CreateDocumentUri(databaseName, collectionName, newEntity.id);
try
{
Document playerDoc = await client.ReadDocumentAsync(docDocumentUri, new RequestOptions { PartitionKey = new PartitionKey(newEntity.id) });
Entity updatedEntity = new Entity
{
id = entity.id,
UserId = entity.UserId,
ItemId = entity.ItemId,
ContentId = entity.ContentId,
EventType = entity.EventType,
OrderId = entity.OrderId,
Region = entity.Region,
SessionId = entity.SessionId,
PartitionId = entity.PartitionId,
Timestamp = entity.Timestamp
};
await client.UpsertDocumentAsync(collectionLink, updatedEntity);
}
catch (DocumentClientException ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
Entity bnewEntity = new Entity
{
UserId = entity.UserId,
ItemId = entity.ItemId,
ContentId = entity.ContentId,
EventType = entity.EventType,
OrderId = entity.OrderId,
Region = entity.Region,
SessionId = entity.SessionId,
PartitionId = entity.PartitionId,
Timestamp = entity.Timestamp
};
await client.UpsertDocumentAsync(collectionLink, bnewEntity);
}
}
}
}
public static async Task <IActionResult> DeleteRecruit(
[HttpTrigger(AuthorizationLevel.Function, "delete", Route = "recruit/{id}")] HttpRequest req,
ILogger log, string id)
{
DocumentClient client = new DocumentClient(new Uri(System.Environment.GetEnvironmentVariable($"CosmosDBEndPoint")), System.Environment.GetEnvironmentVariable($"CosmosDBAuthKey"));
Microsoft.Azure.Documents.Document doc = client.CreateDocumentQuery
(UriFactory.CreateDocumentCollectionUri("recruit-db", "recruits"))
.Where(d => d.Id == id).AsEnumerable().FirstOrDefault();
await client.DeleteDocumentAsync(doc.SelfLink, new RequestOptions { PartitionKey = new PartitionKey(doc.Id) });
return(new OkResult());
}
public static async Task <T> GetItemAsyncInKnowledge(string id, RequestOptions options)
{
try
{
Microsoft.Azure.Documents.Document document = await Constants.Client.ReadDocumentAsync(UriFactory.CreateDocumentUri(Constants.DatabaseIdProd, Constants.CollectionIdProd, id), options);
return((T)(dynamic)document);
}
catch (DocumentClientException e)
{
if (e.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return(null);
}
else
{
throw;
}
}
}
public async Task UpdateDatasetCompressedDetails(Guid datasetId, long zipFileSize, long targzFileSize)
{
var options = new RequestOptions
{
PartitionKey = new PartitionKey(datasetId.ToString())
};
var documentUri = DatasetDocumentUriById(datasetId.ToString());
Microsoft.Azure.Documents.Document dataset = await Client.ReadDocumentAsync(documentUri, options);
if (dataset == null)
{
throw new InvalidOperationException("Could not find dataset document.");
}
dataset.SetPropertyValue("zipFileSize", zipFileSize);
dataset.SetPropertyValue("gzipFileSize", targzFileSize);
dataset.SetPropertyValue("isCompressedAvailable", true);
await Client.ReplaceDocumentAsync(dataset.SelfLink, dataset);
}
public virtual async Task <T> GetItemAsync(string id)
{
try
{
Document document = await Client().ReadDocumentAsync(UriFactory.CreateDocumentUri(DatabaseId, CollectionId, id));
return((T)(dynamic)document);
}
catch (DocumentClientException e)
{
if (e.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return(null);
}
else
{
throw;
}
}
}
public async Task <T> GetAsync(string id)
{
try
{
logger.LogDebug($"Loading item with id {id}");
Document document = await documentClient.ReadDocumentAsync(UriFactory.CreateDocumentUri(databaseId, collectionId, id));
return(ToDocumentBase(document));
}
catch (DocumentClientException e)
{
if (e.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
logger.LogError(e, $"Error loading item with id {id}");
throw;
}
}
private async Task <Microsoft.Azure.Documents.Document> LoadFHIRResourceObject(string databaseName, string collectionName, string identity)
{
try
{
Microsoft.Azure.Documents.Document response = null;
if (!fixeddb)
{
response = await client.ReadDocumentAsync(UriFactory.CreateDocumentUri(databaseName, collectionName, identity), new RequestOptions { PartitionKey = new PartitionKey(identity) });
}
else
{
response = await client.ReadDocumentAsync(UriFactory.CreateDocumentUri(databaseName, collectionName, identity));
}
return(response);
}
catch (Exception de)
{
//Trace.TraceError("Error loading resource: {0}-{1}-{2} Message: {3}",databaseName,collectionName,identity,de.Message);
return(null);
}
}
/// <summary>
/// Updates a single venue document in the collection
/// </summary>
/// <param name="venue">The Venue to update</param>
/// <param name="log">ILogger for logging info/errors</param>
public async Task <ResourceResponse <Document> > UpdateDocAsync(Venue venue, ILogger log)
{
try {
// Get matching venue by Id from the collection
log.LogInformation($"Getting venues from collection with Id {venue?.id}");
Document updated = docClient.CreateDocumentQuery(Collection.SelfLink, new FeedOptions {
EnableCrossPartitionQuery = true, MaxItemCount = -1
})
.Where(u => u.Id == venue.id.ToString())
.AsEnumerable()
.FirstOrDefault();
if (updated == null)
{
return(null);
}
updated.SetPropertyValue("ADDRESS_1", venue.ADDRESS_1);
updated.SetPropertyValue("ADDRESS_2", venue.ADDRESS_2);
updated.SetPropertyValue("COUNTY", venue.COUNTY);
updated.SetPropertyValue("POSTCODE", venue.POSTCODE);
updated.SetPropertyValue("TOWN", venue.TOWN);
updated.SetPropertyValue("VENUE_NAME", venue.VENUE_NAME);
//updated.SetPropertyValue("PHONE", venue.PHONE);
//updated.SetPropertyValue("EMAIL", venue.EMAIL);
//updated.SetPropertyValue("WEBSITE", venue.WEBSITE);
updated.SetPropertyValue("Status", (int)venue.Status);
updated.SetPropertyValue("DateUpdated", DateTime.Now);
updated.SetPropertyValue("Latitude", venue.Latitude);
updated.SetPropertyValue("Longitude", venue.Longitude);
updated.SetPropertyValue("UpdatedBy", venue.UpdatedBy);
return(await docClient.ReplaceDocumentAsync(updated));
} catch (Exception ex) {
throw ex;
}
}
private static Task <ResourceResponse <Document> > GetDocumentForItem <T>(T item)
{
var document = new Document();
// Attempt to provide something close to cosmos and pass the object through using its json property
// names instead (to catch when ID is named differently)
foreach (PropertyInfo propertyInfo in item.GetType().GetProperties())
{
if (propertyInfo.CanRead)
{
var value = propertyInfo.GetValue(item, null);
var jsonProperty =
(JsonPropertyAttribute)propertyInfo.GetCustomAttribute(typeof(JsonPropertyAttribute));
document.SetPropertyValue(jsonProperty?.PropertyName ?? propertyInfo.Name, value);
}
}
var response = ToResourceResponse(document, HttpStatusCode.Accepted);
return(Task.FromResult(response));
}
/// <summary>
/// Inserts a single venue document into the collection
/// </summary>
/// <param name="venue">The Venue to insert</param>
/// <param name="log">ILogger for logging info/errors</param>
public async Task <ResourceResponse <Document> > InsertDocAsync(Venue venue, ILogger log)
{
// Add venue doc to collection
try {
if (venue.id == Guid.Empty)
{
venue.id = Guid.NewGuid();
}
Uri uri = UriFactory.CreateDocumentCollectionUri(SettingsHelper.Database, SettingsHelper.Collection);
// Insert venue doc, get it's cosmos "sequence" value, then write it back into LocationId property
Document doc = await docClient.CreateDocumentAsync(uri, venue);
Sequence sequence = docClient.CreateDocumentQuery <Sequence>(uri, $"SELECT DOCUMENTID(v) AS SequenceId FROM v " +
$"WHERE v.id = \"{venue.id}\"")
.ToList()
.First();
doc.SetPropertyValue("LocationId", sequence.SequenceId + LOCATIONID_OFFSET);
// Add latitude & logitude from onspd azure search
log.LogInformation($"{count++}: getting lat/long for location {venue.POSTCODE}");
SearchParameters parameters = new SearchParameters
{
Select = new[] { "pcds", "lat", "long" },
SearchMode = SearchMode.All,
Top = 1,
QueryType = QueryType.Full
};
DocumentSearchResult <dynamic> results = _onspdIndex.Documents.Search <dynamic>(venue.POSTCODE, parameters);
doc.SetPropertyValue("Latitude", (decimal?)results?.Results?.FirstOrDefault()?.Document?.lat);
doc.SetPropertyValue("Longitude", (decimal?)results?.Results?.FirstOrDefault()?.Document?.@long);
return(await docClient.UpsertDocumentAsync(uri, doc));
} catch (Exception ex) {
throw ex;
}
}
private static async Task QueryPartitionAsync()
{
// Create some documents. Note that creates use the database's self link instead of a specific collection's self link.
// The hash resolver will compute the hash of UserId in order to route the create to either of the collections.
Db.Document johnDocument = await _documentDbClient.CreateDocumentAsync(
_documentDatabase.SelfLink, new UserProfile("J1", "@John"));
Db.Document ryanDocument = await _documentDbClient.CreateDocumentAsync(_documentDatabase.SelfLink,
new UserProfile("U4", "@Ryan"));
var johnProfile = (UserProfile)(dynamic)johnDocument;
// Query for John's document by ID. We can use the PartitionResolver to restrict the query to just the partition containing @John
// Again the query uses the database self link, and relies on the hash resolver to route the appropriate collection.
var query = _documentDbClient.CreateDocumentQuery <UserProfile>(_documentDatabase.SelfLink, null, _rangeResolver.GetPartitionKey(johnProfile))
.Where(u => u.UserName == "@John");
johnProfile = query.AsEnumerable().FirstOrDefault();
// Find the collections where a document exists in. It's uncommon to do this, but can be useful if for example to execute a
// stored procedure against a specific set of partitions.
var collectionLinks = _rangeResolver.ResolveForRead(_rangeResolver.GetPartitionKey(johnProfile)).ToList();
foreach (var collection in collectionLinks)
{
Console.Write("John present in :" + collection);
}
var ryanProfile = (UserProfile)(dynamic)ryanDocument;
collectionLinks = _rangeResolver.ResolveForRead(_rangeResolver.GetPartitionKey(ryanProfile)).ToList();
foreach (var collection in collectionLinks)
{
Console.Write("Ryan present in :" + collection);
}
}
private static T ToDocumentBase(Document document)
{
return((T)(dynamic)document);
}
private static async Task AddDocumentRecord(ILogger log, String name, long length, String text)
{
try
{
// https://nlp.stanford.edu/IR-book/html/htmledition/dropping-common-terms-stop-words-1.html
List <String> stopWords = new List <string>();
stopWords.Add("a");
stopWords.Add("an");
stopWords.Add("and");
stopWords.Add("are");
stopWords.Add("as");
stopWords.Add("at");
stopWords.Add("be");
stopWords.Add("by");
stopWords.Add("for");
stopWords.Add("from");
stopWords.Add("has");
stopWords.Add("he");
stopWords.Add("in");
stopWords.Add("is");
stopWords.Add("it");
stopWords.Add("its");
stopWords.Add("of");
stopWords.Add("on");
stopWords.Add("that");
stopWords.Add("the");
stopWords.Add("to");
stopWords.Add("was");
stopWords.Add("were");
stopWords.Add("will");
stopWords.Add("with");
client = new DocumentClient(new Uri(EndpointUrl), PrimaryKey);
// Create database and collection if necessary
await client.CreateDatabaseIfNotExistsAsync(new Database { Id = "SearchDB" });
await client.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri("SearchDB"), new DocumentCollection { Id = "DocCollection" }, new RequestOptions { PartitionKey = new PartitionKey("/Category"), OfferThroughput = 400 });
await client.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri("SearchDB"), new DocumentCollection { Id = "DocWordCollection" }, new RequestOptions { PartitionKey = new PartitionKey("/DocId"), OfferThroughput = 400 });
// From extracted text, create a collection of unique words
String[] items = text.ToLower().Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
int wordCount = items.Length;
IEnumerable <String> uniqueWords = items.Distinct();
int uniqueWordCount = uniqueWords.Count();
// Add Document record
String docType = null;
int pos = name.LastIndexOf(".");
if (pos != -1)
{
docType = name.Substring(pos + 1);
}
Document document = new Document()
{
Category = "docs",
DocId = name,
Name = name,
DocType = docType,
Owner = "David Pallmann",
Size = Convert.ToInt32(length),
Text = text,
Words = text.Split(' ').Length + 1
};
try
{
// Partition key provided either doesn't correspond to definition in the collection or doesn't match partition key field values specified in the document.
log.LogInformation("Creating document - Category: " + document.Category + ", DocId: " + document.DocId); // + name);
await client.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri("SearchDB", "DocCollection"), document, new RequestOptions()
{
PartitionKey = new PartitionKey("docs")
});
log.LogInformation("Document record created, Id: " + document.DocId);
}
catch (DocumentClientException de)
{
try
{
log.LogInformation("ERROR creating document: " + de.GetType().Name + ": " + de.Message);
// Create document failed, so perform a Replace instead
log.LogInformation("Document exists, Replacing existing document");
var docCollectionUrl = UriFactory.CreateDocumentCollectionUri("SearchDB", "DocCollection");
var docCollection = (await client.ReadDocumentCollectionAsync(docCollectionUrl, new RequestOptions()
{
PartitionKey = new PartitionKey("docs")
})).Resource;
var query = new SqlQuerySpec("SELECT * FROM DocCollection doc WHERE doc.DocId = @DocId",
new SqlParameterCollection(new SqlParameter[] { new SqlParameter {
Name = "@DocId", Value = document.DocId
} }));
var existingDocRecords = client.CreateDocumentQuery <Microsoft.Azure.Documents.Document>(docCollectionUrl, query, new FeedOptions()
{
EnableCrossPartitionQuery = true
}).AsEnumerable();
if (existingDocRecords != null && existingDocRecords.Count() > 0)
{
Microsoft.Azure.Documents.Document doc = existingDocRecords.First <Microsoft.Azure.Documents.Document>();
doc.SetPropertyValue("Category", document.Category);
doc.SetPropertyValue("DocId", document.DocId);
doc.SetPropertyValue("Name", document.Name);
doc.SetPropertyValue("DocType", document.DocType);
doc.SetPropertyValue("Owner", document.Owner);
doc.SetPropertyValue("Text", document.Text);
doc.SetPropertyValue("Words", document.Words);
await client.ReplaceDocumentAsync(existingDocRecords.First <Microsoft.Azure.Documents.Document>().SelfLink, doc, new RequestOptions()
{
PartitionKey = new PartitionKey("docs")
});
log.LogInformation("Document record replaced, Id: " + document.DocId);
}
}
catch (DocumentClientException de2)
{
log.LogInformation("ERROR replacing document: " + de2.GetType().Name + ": " + de2.Message);
}
}
var collUrl = UriFactory.CreateDocumentCollectionUri("SearchDB", "DocWordCollection");
var doc1 = (await client.ReadDocumentCollectionAsync(collUrl, new RequestOptions()
{
PartitionKey = new PartitionKey(document.DocId)
})).Resource;
var existingDocWordRecords = client.CreateDocumentQuery(doc1.SelfLink, new FeedOptions()
{
PartitionKey = new PartitionKey(document.DocId)
}).AsEnumerable().ToList();
if (existingDocWordRecords != null)
{
int count = 0;
try
{
RequestOptions options = new RequestOptions()
{
PartitionKey = new PartitionKey(document.DocId)
};
log.LogInformation("Deleting prior DocWord records...");
foreach (Microsoft.Azure.Documents.Document word in existingDocWordRecords)
{
if (LogDetail)
{
log.LogInformation("Found document SelfLink: " + word.SelfLink + ", DocId: " + word.GetPropertyValue <String>("DocId") + ", Word: " + word.GetPropertyValue <String>("Word"));
}
await client.DeleteDocumentAsync(word.SelfLink /* UriFactory.CreateDocumentUri("SearchDB", "DocWordCollection", word.SelfLink) */, options); //, options);
count++;
}
}
catch (DocumentClientException de)
{
log.LogInformation("ERROR deleting DocWord record: " + de.Message);
}
catch (Exception ex)
{
log.LogInformation("EXCEPTION deleting DocWord record: " + ex.GetType().Name + ": " + ex.Message);
if (ex.InnerException != null)
{
log.LogInformation("INNER EXCEPTION: " + ex.InnerException.GetType().Name + ": " + ex.InnerException.Message);
}
}
log.LogInformation(count.ToString() + " DocWord records deleted");
}
// Store document words in Words collection
try
{
log.LogInformation("Adding DocWord records with partition key " + document.DocId + "...");
int count = 0;
DocWord docWord = null;
foreach (String word in uniqueWords)
{
if (!stopWords.Contains(word))
{
docWord = new DocWord()
{
Id = Guid.NewGuid().ToString(),
DocId = document.DocId,
Word = word
};
if (LogDetail)
{
log.LogInformation("About to: CreateDocumentAsync on DocWordCollection: word: " + docWord.Word + ", DocId:" + docWord.DocId);
}
await client.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri("SearchDB", "DocWordCollection"), docWord, new RequestOptions()
{
PartitionKey = new PartitionKey(document.DocId)
});
count++;
}
}
log.LogInformation(count.ToString() + " DocWord records created");
}
catch (DocumentClientException de)
{
log.LogInformation("ERROR creating DocWord record: " + de.Message);
}
}
catch (DocumentClientException de)
{
Exception baseException = de.GetBaseException();
log.LogInformation("{0} error occurred: {1}, Message: {2}", de.StatusCode, de.Message, baseException.Message);
}
catch (Exception e)
{
Exception baseException = e.GetBaseException();
log.LogInformation("Error: {0}, Message: {1}", e.Message, baseException.Message);
}
}
/// <summary>
/// Delete the current document
/// </summary>
/// <param name="document">The document entity to delete.</param>
/// <returns>The async task.</returns>
public async Task <ResourceResponse <Microsoft.Azure.Documents.Document> > DeleteDocumentAsync(Microsoft.Azure.Documents.Document document)
{
return(await _client.DeleteDocumentAsync(document.SelfLink));
}
