public static bool AddRegexMatchs(DataConfig providerConfig, List <CaptureRegexMatch> matches) { bool isSaved = false; switch (providerConfig.ProviderType) { case "azure.tableservice": List <TableOperation> ops = new List <TableOperation>(); CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.RecordAssociationRegexMatch); foreach (POCO.CaptureRegexMatch match in matches) { AzureRegexMatch az = new AzureRegexMatch(match); TableOperation operation = TableOperation.InsertOrReplace(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); } break; case "internal.mongodb": IMongoCollection <MongoRegexMatchUpsert> collection = Utils.GetMongoCollection <MongoRegexMatchUpsert>(providerConfig, MongoTableNames.RecordAssociationRegexMatch); foreach (POCO.CaptureRegexMatch match in matches) { MongoRegexMatchUpsert mongoObject = Utils.ConvertType <MongoRegexMatchUpsert>(match); // Create the update filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(mongoObject.PartitionKey), "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(mongoObject.RowKey), "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoRegexMatchUpsert> filter = Utils.GenerateMongoFilter <MongoRegexMatchUpsert>(filters); // Create the upsert options MongoDB.Driver.ReplaceOptions options = new ReplaceOptions(); options.IsUpsert = true; // Upsert collection.ReplaceOne(filter, mongoObject, options); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } isSaved = true; return(isSaved); }
public static List <POCO.FileProcessException> GetFileProcessException(DataConfig providerConfig, POCO.System system, string exceptionType) { string tableName = string.Empty; List <POCO.FileProcessException> docs = new List <POCO.FileProcessException>(); // Create filter List <Filter> filters = new List <Filter>(); Filter pkfilt = new Filter("PartitionKey", Utilities.Converters.CleanTableKey(exceptionType), "eq"); filters.Add(pkfilt); switch (providerConfig.ProviderType) { case ProviderType.Azure: // Calcuate table name based on System Id and file process result (partitionkey) tableName = "stlpfileexception-" + Utilities.Converters.CleanTableKey(exceptionType) + "-" + system.SystemId.ToString(); string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureFileProcessException> azdata = new List <AzureFileProcessException>(); AzureTableAdaptor <AzureFileProcessException> adaptor = new AzureTableAdaptor <AzureFileProcessException>(); azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter); foreach (var doc in azdata) { docs.Add(doc.Value); } break; case ProviderType.Mongo: // Calcuate table name based on System Id and file process result (partitionkey) tableName = "fileexception-" + Utilities.Converters.CleanTableKey(exceptionType) + "-" + system.SystemId.ToString(); var collection = Utils.GetMongoCollection <MongoFileProcessException_read>(providerConfig, tableName); FilterDefinition <MongoFileProcessException_read> filter = Utils.GenerateMongoFilter <MongoFileProcessException_read>(filters); var documents = collection.Find(filter).ToList(); foreach (var doc in documents) { docs.Add(doc); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(docs); }
public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId) { nextPageId = string.Empty; List <POCO.SakaiFile> files = new List <POCO.SakaiFile>(); switch (providerConfig.ProviderType) { case "azure.tableservice": throw new NotImplementedException(); break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, MongoTableNames.SakaiFiles); // Add an _id filter if a page has been requested if (thisPageId != null && thisPageId != string.Empty) { filters.Insert(0, new Filter("_id", thisPageId, "gt")); } FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters); //DEBUG output the filter values //foreach (Castlepoint.DataFactory.Filter debugFilter in filters) //{ // // Output the filter field names and values // Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue); //} var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList(); foreach (var sakaifile in documents) { files.Add(sakaifile); } // Get the next page id if (documents.Count == rowLimit) { // Set the next page id nextPageId = documents[documents.Count - 1]._id.ToString(); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(files); }
public static List <POCO.BasecampProjectEntity> GetProjects(DataConfig providerConfig, POCO.System system, POCO.BasecampEntity basecampProject) { List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq"); Filter rkFilter = new Filter("RowKey", Utils.CleanTableKey(basecampProject.url), "eq"); filters.Add(pkFilter); filters.Add(rkFilter); return(BasecampProject.GetProjects(providerConfig, filters)); }
public static bool ForceRescan(DataConfig providerConfig, POCO.SakaiFile file) { POCO.FileBatchStatus batchstatus = new FileBatchStatus(file.PartitionKey, file.RowKey); batchstatus.BatchGuid = Guid.Empty; batchstatus.BatchStatus = string.Empty; batchstatus.JsonFileProcessResult = "{}"; DataFactory.File.UpdateFileBatchStatus(providerConfig, batchstatus, "ntfsfiles"); return(true); }
public static List <POCO.LogClassification> GetLogClassification(DataConfig providerConfig, List <Filter> filters, string tableNameSuffix_yyyymm) { // Validate the YYYYMM suffix if (tableNameSuffix_yyyymm.Length != 6) { throw new ApplicationException("Invalid size for table suffix, must be YYYYMM"); } string tableName = string.Empty; List <POCO.LogClassification> docs = new List <POCO.LogClassification>(); switch (providerConfig.ProviderType) { case ProviderType.Azure: tableName = AzureTableNames.LogClassification + tableNameSuffix_yyyymm; string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureLogClassification> azdata = new List <AzureLogClassification>(); AzureTableAdaptor <AzureLogClassification> adaptor = new AzureTableAdaptor <AzureLogClassification>(); azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter); foreach (var doc in azdata) { docs.Add(doc.Value); } break; case ProviderType.Mongo: tableName = MongoTableNames.LogClassification + tableNameSuffix_yyyymm; var collection = Utils.GetMongoCollection <MongoLogClassification_Read>(providerConfig, tableName); FilterDefinition <MongoLogClassification_Read> filter = Utils.GenerateMongoFilter <MongoLogClassification_Read>(filters); var documents = collection.Find(filter).ToList(); foreach (var doc in documents) { docs.Add(doc); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(docs); }
public static string AddSite(DataConfig providerConfig, POCO.System system, POCO.SakaiSite sourceSite) { POCO.SakaiSiteEntity siteEntity = new SakaiSiteEntity(Utils.CleanTableKey(system.PartitionKey), Utils.CleanTableKey(sourceSite.SITE_ID)); siteEntity.CPStatus = string.Empty; siteEntity.ItemCount = 0; siteEntity.Name = sourceSite.TITLE; siteEntity.ServerRelativeUrl = sourceSite.SITE_ID; siteEntity.TimeCreated = sourceSite.CREATEDON.ToUniversalTime(); siteEntity.TimeLastModified = sourceSite.MODIFIEDON.ToUniversalTime(); return(SakaiSite.AddSite(providerConfig, siteEntity)); }
public static List <POCO.SakaiSiteEntity> GetSites(DataConfig providerConfig, POCO.System system, POCO.SakaiSite site) { List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq"); filters.Add(pkFilter); Filter rkFilter = new Filter("RowKey", Utils.CleanTableKey(site.SITE_ID), "eq"); filters.Add(rkFilter); return(SakaiSite.GetSites(providerConfig, filters)); }
public static List <POCO.RecordAuthorityFilter> GetAuthorityFilters(DataConfig providerConfig, POCO.RecordAuthorityFilter rafilt) { List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", rafilt.PartitionKey, "eq"); filters.Add(pkFilter); Filter rkFilter = new Filter("RowKey", rafilt.RowKey, "eq"); filters.Add(rkFilter); return(GetAssignedForSystem(providerConfig, filters)); }
public static string AddProject(DataConfig providerConfig, POCO.System system, POCO.BasecampEntity sourceProject) { POCO.BasecampProjectEntity projectEntity = new BasecampProjectEntity(Utils.CleanTableKey(system.PartitionKey), Utils.CleanTableKey(sourceProject.url)); projectEntity.CPStatus = string.Empty; projectEntity.ItemCount = 0; projectEntity.Name = sourceProject.name; projectEntity.ServerRelativeUrl = sourceProject.url; projectEntity.TimeCreated = sourceProject.created_at.ToUniversalTime(); projectEntity.TimeLastModified = sourceProject.updated_at.ToUniversalTime(); return(BasecampProject.AddProject(providerConfig, projectEntity)); }
public static bool UpdateRecordConfig(DataConfig providerConfig, POCO.System system, string config) { bool isUpdatedOk = false; POCO.RecordConfigUpdate updateRecordCfg = new RecordConfigUpdate(); updateRecordCfg.PartitionKey = system.PartitionKey; updateRecordCfg.RowKey = system.RowKey; updateRecordCfg.JsonRecordIdentificationConfig = config; switch (providerConfig.ProviderType) { case "azure.tableservice": AzureRecordIdentificationConfig az = new AzureRecordIdentificationConfig(updateRecordCfg); CloudTable table = Utils.GetCloudTable(providerConfig, System.TableNames.System); TableOperation operation = TableOperation.InsertOrMerge(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); isUpdatedOk = true; break; case "internal.mongodb": IMongoCollection <MongoRecordIdentificationConfig> collection = Utils.GetMongoCollection <MongoRecordIdentificationConfig>(providerConfig, System.TableNames.System); //MongoSystemStatUpdate mongoObject = Utils.ConvertType<MongoSystemStatUpdate>(systemStat); // Create the update filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", updateRecordCfg.PartitionKey, "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", updateRecordCfg.RowKey, "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoRecordIdentificationConfig> filter = Utils.GenerateMongoFilter <MongoRecordIdentificationConfig>(filters); var update = Builders <MongoRecordIdentificationConfig> .Update .Set("JsonRecordIdentificationConfig", updateRecordCfg.JsonRecordIdentificationConfig); // Update the batch status UpdateResult result = collection.UpdateOne(filter, update); isUpdatedOk = true; break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(isUpdatedOk); }
public static string AddFileProcessingException(DataConfig providerConfig, POCO.System system, POCO.FileProcessException fileException) { string tableName = string.Empty; switch (providerConfig.ProviderType) { case "azure.tableservice": // Calcuate table name based on System Id and file process result (partitionkey) tableName = "stlpfileexception-" + fileException.PartitionKey + "-" + system.SystemId.ToString(); AzureFileProcessException az = new AzureFileProcessException(fileException); CloudTable table = Utils.GetCloudTable(providerConfig, tableName); TableOperation operation = TableOperation.InsertOrReplace(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); break; case "internal.mongodb": // Calcuate table name based on System Id and file process result (partitionkey) tableName = "fileexception-" + fileException.PartitionKey + "-" + system.SystemId.ToString(); IMongoCollection <MongoFileProcessException> collection = Utils.GetMongoCollection <MongoFileProcessException>(providerConfig, tableName); MongoFileProcessException mongoObject = Utils.ConvertType <MongoFileProcessException>(fileException); // Create the upsert filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", fileException.PartitionKey, "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", fileException.RowKey, "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoFileProcessException> filter = Utils.GenerateMongoFilter <MongoFileProcessException>(filters); // Create the upsert options MongoDB.Driver.ReplaceOptions options = new ReplaceOptions(); options.IsUpsert = true; // Upsert collection.ReplaceOne(filter, mongoObject, options); return(string.Empty); default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } //TODO return id of new object if supported return(string.Empty); }
public static bool SetConfigEnabled(DataConfig providerConfig, string systemPartitionKey, string systemRowKey, bool enableSystem) { bool isUpdatedOk = false; POCO.SystemEnabledUpdate updateEnabled = new POCO.SystemEnabledUpdate(); updateEnabled.PartitionKey = systemPartitionKey; updateEnabled.RowKey = systemRowKey; updateEnabled.Enabled = enableSystem; switch (providerConfig.ProviderType) { case "azure.tableservice": AzureSystemEnabledUpdate az = new AzureSystemEnabledUpdate(updateEnabled); CloudTable table = Utils.GetCloudTable(providerConfig, System.TableNames.System); TableOperation operation = TableOperation.InsertOrMerge(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); isUpdatedOk = true; break; case "internal.mongodb": IMongoCollection <MongoSystemEnabledUpdate> collection = Utils.GetMongoCollection <MongoSystemEnabledUpdate>(providerConfig, System.TableNames.System); // Create the update filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", updateEnabled.PartitionKey, "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", updateEnabled.RowKey, "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoSystemEnabledUpdate> filter = Utils.GenerateMongoFilter <MongoSystemEnabledUpdate>(filters); var update = Builders <MongoSystemEnabledUpdate> .Update .Set("Enabled", updateEnabled.Enabled); // Update the batch status UpdateResult result = collection.UpdateOne(filter, update); isUpdatedOk = true; break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(isUpdatedOk); }
public static string AddMessage(DataConfig providerConfig, POCO.System system, POCO.GFIMailbox mailbox, POCO.GFIMessage message) { string partitionKey = system.PartitionKey; POCO.GFIMessageEntity docEntity = new GFIMessageEntity(Utils.CleanTableKey(partitionKey), Utils.CleanTableKey(message.MarcId.ToString())); docEntity.CPStatus = string.Empty; docEntity.ItemCount = 0; docEntity.Name = message.Subject; docEntity.ServerRelativeUrl = string.Empty; docEntity.TimeCreated = message.SentDate.ToUniversalTime(); docEntity.TimeLastModified = message.ReceivedDate.ToUniversalTime(); return(GFIArchiverMailbox.AddMessage(providerConfig, docEntity)); }
public static void UpdateFileBatchStatus(DataConfig providerConfig, POCO.FileBatchStatus fileBatchStatus) { switch (providerConfig.ProviderType) { case "azure.tableservice": DataFactory.File.UpdateFileBatchStatus(providerConfig, fileBatchStatus, AzureTableNames.EOMessage); break; case "internal.mongodb": DataFactory.File.UpdateFileBatchStatus(providerConfig, fileBatchStatus, MongoTableNames.EOMessage); break; } return; }
public static void Delete(DataConfig providerConfig, POCO.ProcessingBatchStatus processingBatchStatus) { List <Filter> filters = new List <Filter>(); Filter pk = new Filter("PartitionKey", processingBatchStatus.PartitionKey, "eq"); filters.Add(pk); Filter rk = new Filter("RowKey", processingBatchStatus.RowKey, "eq"); filters.Add(rk); Filter typekey = new Filter("ProcessType", processingBatchStatus.ProcessType, "eq"); filters.Add(typekey); switch (providerConfig.ProviderType) { case "azure.tableservice": AzureProcessingBatchStatus az = new AzureProcessingBatchStatus(processingBatchStatus); az.ETag = "*"; CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.SystemProcessingStatus); TableOperation operation = TableOperation.Delete(az); Task <TableResult> tDelete = table.ExecuteAsync(operation); tDelete.Wait(); // Check for "success with no status" code if (tDelete.Result.HttpStatusCode != 204) { // TODO bool isNotDeleted = true; } break; case "internal.mongodb": FilterDefinition <BsonDocument> filter = Utils.GenerateMongoFilter <BsonDocument>(filters); // Delete the rows IMongoCollection <BsonDocument> collection = Utils.GetMongoCollection <BsonDocument>(providerConfig, MongoTableNames.SystemProcessingStatus); DeleteResult result = collection.DeleteMany(filter); return; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return; }
public static string AddFile(DataConfig providerConfig, POCO.System system, POCO.BasecampEntity sourceProject, POCO.BasecampDocument sourceDocument) { string partitionKey = GetBasecampBucketUrl(sourceProject.url); POCO.BasecampDocumentEntity docEntity = new BasecampDocumentEntity(Utils.CleanTableKey(partitionKey), Utils.CleanTableKey(sourceDocument.url)); docEntity.CPStatus = string.Empty; docEntity.ItemCount = 0; docEntity.Name = sourceDocument.title; docEntity.ServerRelativeUrl = sourceDocument.url; docEntity.TimeCreated = sourceDocument.created_at.ToUniversalTime(); docEntity.TimeLastModified = sourceDocument.updated_at.ToUniversalTime(); docEntity.Title = sourceDocument.title; return(BasecampProject.AddFile(providerConfig, docEntity)); }
public static void AddFileBatchStatus(DataConfig providerConfig, POCO.FileBatchStatus fileBatchStatus) { switch (providerConfig.ProviderType) { case "azure.tableservice": DataFactory.File.AddFileBatchStatus(providerConfig, fileBatchStatus, AzureTableNames.EOMessageBatchStatus); break; case "internal.mongodb": DataFactory.File.AddFileBatchStatus(providerConfig, fileBatchStatus, MongoTableNames.EOMessageBatchStatus); break; } //TODO return id of new object if supported return; }
public static List <POCO.CaptureRegexMatch> GetRegexMatches(DataConfig providerConfig, POCO.System system, string fileUri) { // Create a filter to match the Ontology provided List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", system.PartitionKey, "eq"); filters.Add(pkFilter); string rowkey = fileUri; Filter rk = new Filter("RowKey", rowkey, "ge"); filters.Add(rk); rk = new Filter("RowKey", Utils.GetLessThanFilter(rowkey), "lt"); filters.Add(rk); return(GetRegexMatches(providerConfig, filters)); }
public static long PATCH003_UpdateSakaiFileItemUri(DataConfig providerConfig, POCO.SakaiFile file) { switch (providerConfig.ProviderType) { case "azure.tableservice": throw new NotImplementedException(); break; case "internal.mongodb": IMongoCollection <MongoSakaiUpdatePatch003> collection = Utils.GetMongoCollection <MongoSakaiUpdatePatch003>(providerConfig, MongoTableNames.SakaiFiles); // Create the update filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(file.PartitionKey), "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(file.RowKey), "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoSakaiUpdatePatch003> filter = Utils.GenerateMongoFilter <MongoSakaiUpdatePatch003>(filters); //string updateParam = "{$set: {ItemUri: '" + file.SourceRelativeUrl.ToString() + "'}}"; //BsonDocument updateDoc = BsonDocument.Parse(updateParam); var updateDoc = Builders <MongoSakaiUpdatePatch003> .Update .Set("ItemUri", file.SourceRelativeUrl.ToString()); // Update the batch status UpdateResult result = collection.UpdateOne(filter, updateDoc); if (result.IsAcknowledged && result.IsModifiedCountAvailable) { return(result.ModifiedCount); } else { return(0); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(0); }
public static List <POCO.OntologyTermMatchResults> GetOntologyMatchSummary(DataConfig providerConfig, POCO.OntologyAssigned ontologyAssigned) { // Get the ontology terms for this ontology List <POCO.OntologyTerm> oTerms = DataFactory.Ontology.GetOntologyTerms(providerConfig, ontologyAssigned); // Convert the ontology terms into match results List <POCO.OntologyTermMatchResults> results = new List <POCO.OntologyTermMatchResults>(); foreach (POCO.OntologyTerm term in oTerms) { // Create a matching result object POCO.OntologyTermMatchResults result = new POCO.OntologyTermMatchResults(); result.PartitionKey = term.PartitionKey; result.RowKey = term.RowKey; result.Term = term.Term; result.ParentTerm = term.ParentTerm; results.Add(result); } // Get all the ontology term matches for this ontology List <POCO.OntologyTermMatch> termMatches = DataFactory.Ontology.GetOntologyTermMatches(providerConfig, ontologyAssigned.RowKey); foreach (POCO.OntologyTermMatch match in termMatches) { // Find the Term Match Result for this Term Match POCO.OntologyTermMatchResults matchedResult = results.Find(r => r.PartitionKey == match.PartitionKey && r.RowKey == match.TermRowKey); if (matchedResult != null) { matchedResult.NumRecordAssociationMatches++; } } //TODO use Linq to select all the parent nodes instead // Walk the ontology tree and sum child counts to parent counts for (int i = 0; i < results.Count; i++) { POCO.OntologyTermMatchResults result = results[i]; // Find a parent ontology term (which has no ParentTerm = OntologyUri (PartitionKey)) if (result.ParentTerm == result.PartitionKey) { // Recursively sum the child terms up to its parent SumOntologySummary(ref result, ref results); } } return(results); }
public static void UpdateSentenceStat(DataConfig providerConfig, POCO.System system, List <POCO.SentenceStat> sentenceStats) { switch (providerConfig.ProviderType) { case "azure.tableservice": AzureSentenceStatUpdate az = new AzureSentenceStatUpdate(system, sentenceStats); CloudTable table = Utils.GetCloudTable(providerConfig, System.TableNames.System); TableOperation operation = TableOperation.InsertOrMerge(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); break; case "internal.mongodb": IMongoCollection <MongoSystemStatUpdate> collection = Utils.GetMongoCollection <MongoSystemStatUpdate>(providerConfig, "stlpsystems"); //MongoSystemStatUpdate mongoObject = Utils.ConvertType<MongoSystemStatUpdate>(systemStat); // Create the update filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq"); //DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(system.RowKey), "eq"); filters.Add(pkFilter); //filters.Add(rkFilter); FilterDefinition <MongoSystemStatUpdate> filter = Utils.GenerateMongoFilter <MongoSystemStatUpdate>(filters); // Serialize the stats object string jsonStatsSerialized = JsonConvert.SerializeObject(sentenceStats); //string updateParam = "{$set: {JsonSentenceStats: '" + jsonStatsSerialized + "'}}"; //BsonDocument updateDoc = BsonDocument.Parse(updateParam); var update = Builders <MongoSystemStatUpdate> .Update .Set("JsonSentenceStats", jsonStatsSerialized); // Update the batch status UpdateResult result = collection.UpdateOne(filter, update); return; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return; }
public static POCO.OntologyMatchStatus GetOntologyMatchStatus(DataConfig providerConfig, RecordToRecordAssociation recassoc, OntologyAssigned ontology) { POCO.OntologyMatchStatus matchStatus = null; // Create a filter to match the Ontology provided List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", ontology.RowKey, "eq"); filters.Add(pkFilter); Filter rkFilter = new Filter("RowKey", recassoc.RowKey, "eq"); filters.Add(rkFilter); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureOntologyMatchStatus> azs = new List <AzureOntologyMatchStatus>(); AzureTableAdaptor <AzureOntologyMatchStatus> az = new AzureTableAdaptor <AzureOntologyMatchStatus>(); azs = az.ReadTableData(providerConfig, AzureTableNames.OntologyMatchStatus, combinedFilter); if (azs.Count > 0) { matchStatus = azs[0].Value; } return(matchStatus); case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoOntologyMatchStatus>(providerConfig, MongoTableNames.OntologyMatchStatus); FilterDefinition <MongoOntologyMatchStatus> filter = Utils.GenerateMongoFilter <MongoOntologyMatchStatus>(filters); var documents = collection.Find(filter).ToList(); if (documents.Count > 0) { matchStatus = documents[0]; } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(matchStatus); }
public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters) { List <POCO.SakaiFile> files = new List <POCO.SakaiFile>(); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureSakaiFile> azdata = new List <AzureSakaiFile>(); AzureTableAdaptor <AzureSakaiFile> adaptor = new AzureTableAdaptor <AzureSakaiFile>(); azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFiles, combinedFilter); foreach (var doc in azdata) { files.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, SakaiResource.MongoTableNames.SakaiFiles); FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters); //DEBUG output the filter values //foreach (Castlepoint.DataFactory.Filter debugFilter in filters) //{ // // Output the filter field names and values // Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue); //} var documents = collection.Find(filter).ToList(); foreach (var SakaiFile in documents) { files.Add(SakaiFile); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(files); }
public static string AddFileProcessingLog(DataConfig providerConfig, POCO.LogFileProcessing log) { switch (providerConfig.ProviderType) { case "azure.tableservice": // Cycle the tables each month to manage size string tableNameSuffix = DateTime.UtcNow.ToString(Utils.TableSuffixDateFormatYM); string tableName = "stlplogfileprocessresult" + tableNameSuffix; AzureLogFileProcessing az = new AzureLogFileProcessing(log); CloudTable table = Utils.GetCloudTable(providerConfig, tableName); TableOperation operation = TableOperation.InsertOrReplace(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); break; case "internal.mongodb": IMongoCollection <MongoLogFileProcessing> collection = Utils.GetMongoCollection <MongoLogFileProcessing>(providerConfig, "logfileprocessing"); MongoLogFileProcessing mongoObject = Utils.ConvertType <MongoLogFileProcessing>(log); // Create the upsert filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", log.PartitionKey, "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", log.RowKey, "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoLogFileProcessing> filter = Utils.GenerateMongoFilter <MongoLogFileProcessing>(filters); // Create the upsert options MongoDB.Driver.ReplaceOptions options = new ReplaceOptions(); options.IsUpsert = true; // Upsert collection.ReplaceOne(filter, mongoObject, options); return(string.Empty); default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } //TODO return id of new object if supported return(string.Empty); }
public static List <POCO.RecordAuthorityFilter> GetAssignedForRecord(DataConfig providerConfig, POCO.Record record) { List <POCO.RecordAuthorityFilter> assignedrecordauthoritys = new List <POCO.RecordAuthorityFilter>(); List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", record.RowKey, "eq"); filters.Add(pkFilter); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureRecordAuthorityFilter> azdata = new List <AzureRecordAuthorityFilter>(); AzureTableAdaptor <AzureRecordAuthorityFilter> adaptor = new AzureTableAdaptor <AzureRecordAuthorityFilter>(); azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordsAssignedRecordsAuthority, combinedFilter); foreach (var doc in azdata) { assignedrecordauthoritys.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoRecordAuthorityFilter>(providerConfig, MongoTableNames.RecordsAssignedRecordsAuthority); FilterDefinition <MongoRecordAuthorityFilter> filter = Utils.GenerateMongoFilter <MongoRecordAuthorityFilter>(filters); var documents = collection.Find(filter).ToList(); foreach (var assignedra in documents) { assignedrecordauthoritys.Add(assignedra); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(assignedrecordauthoritys); }
public static List <POCO.OntologyAssigned> GetRecordAssignedOntology(DataConfig providerConfig, POCO.Record record) { List <POCO.OntologyAssigned> ontologies = new List <POCO.OntologyAssigned>(); List <Filter> filters = new List <Filter>(); Filter systemFilter = new Filter("PartitionKey", Utils.CleanTableKey(record.PartitionKey), "eq"); filters.Add(systemFilter); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureOntologyAssigned> azs = new List <AzureOntologyAssigned>(); AzureTableAdaptor <AzureOntologyAssigned> az = new AzureTableAdaptor <AzureOntologyAssigned>(); azs = az.ReadTableData(providerConfig, AzureTableNames.RecordsAssignedOntology, combinedFilter); foreach (var doc in azs) { ontologies.Add(doc.Value); } return(ontologies); case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoOntologyAssigned>(providerConfig, MongoTableNames.RecordsAssignedOntology); FilterDefinition <MongoOntologyAssigned> filter = Utils.GenerateMongoFilter <MongoOntologyAssigned>(filters); var documents = collection.Find(filter).ToList(); foreach (var ontology in documents) { ontologies.Add(ontology); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(ontologies); }
public static List <POCO.OntologyTerm> GetOntologyTerms(DataConfig providerConfig, string ontologyUri) { List <POCO.OntologyTerm> ontologyTerms = new List <POCO.OntologyTerm>(); // Create a filter to match the Ontology provided List <Filter> filters = new List <Filter>(); Filter pkFilter = new Filter("PartitionKey", ontologyUri, "eq"); filters.Add(pkFilter); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureOntologyTerm> azs = new List <AzureOntologyTerm>(); AzureTableAdaptor <AzureOntologyTerm> az = new AzureTableAdaptor <AzureOntologyTerm>(); azs = az.ReadTableData(providerConfig, AzureTableNames.OntologyTerms, combinedFilter); foreach (var doc in azs) { ontologyTerms.Add(doc.Value); } return(ontologyTerms); case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoOntologyTerm>(providerConfig, MongoTableNames.OntologyTerms); FilterDefinition <MongoOntologyTerm> filter = Utils.GenerateMongoFilter <MongoOntologyTerm>(filters); var documents = collection.Find(filter).ToList(); foreach (var ontology in documents) { ontologyTerms.Add(ontology); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(ontologyTerms); }
//public static List<POCO.O365.SPFolder> GetFolders(DataConfig providerConfig, List<Filter> filters) //{ // List<POCO.O365.SPFolder> webInfo = new List<POCO.O365.SPFolder>(); // switch (providerConfig.ProviderType) // { // case "azure.tableservice": // string combinedFilter = Utils.GenerateAzureFilter(filters); // List<AzureSPFolder> azdata = new List<AzureSPFolder>(); // AzureTableAdaptor<AzureSPFolder> adaptor = new AzureTableAdaptor<AzureSPFolder>(); // azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.SPFolder, combinedFilter); // foreach (var doc in azdata) // { // webInfo.Add(doc.Value); // } // break; // case "internal.mongodb": // var collection = Utils.GetMongoCollection<MongoSPFolder>(providerConfig, MongoTableNames.SPFolder); // FilterDefinition<MongoSPFolder> filter = Utils.GenerateMongoFilter<MongoSPFolder>(filters); // //TODO paging // var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(1000).ToList(); // foreach (var doc in documents) // { // webInfo.Add(doc); // } // break; // default: // throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); // } // return webInfo; //} //public static void UpdateSPOWebInfoLastProcessed(DataConfig providerConfig, POCO.O365.SPOWebInfoEntity webInfo) //{ // switch (providerConfig.ProviderType) // { // case "azure.tableservice": // AzureSPOWebInfoEntity az = new AzureSPOWebInfoEntity(webInfo); // CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.SPOTracking); // TableOperation operation = TableOperation.InsertOrMerge(az); // Task tUpdate = table.ExecuteAsync(operation); // tUpdate.Wait(); // break; // case "internal.mongodb": // IMongoCollection<MongoSPOWebInfoEntity> collection = Utils.GetMongoCollection<MongoSPOWebInfoEntity>(providerConfig, MongoTableNames.SPOTracking); // MongoSPOWebInfoEntity mongoObject = Utils.ConvertType<MongoSPOWebInfoEntity>(webInfo); // // Create the update filter // List<DataFactory.Filter> filters = new List<DataFactory.Filter>(); // DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(mongoObject.PartitionKey), "eq"); // DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(mongoObject.RowKey), "eq"); // filters.Add(pkFilter); // filters.Add(rkFilter); // FilterDefinition<MongoSPOWebInfoEntity> filter = Utils.GenerateMongoFilter<MongoSPOWebInfoEntity>(filters); // var update = Builders<MongoSPOWebInfoEntity>.Update // .Set("LastItemModifiedDate", webInfo.LastItemModifiedDate) // .Set("LastItemUserModifiedDate", webInfo.LastItemUserModifiedDate); // // Update the batch status // UpdateResult result = collection.UpdateOne(filter, update); // return; // default: // throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); // } // return; //} public static void UpdateEOFolder(DataConfig providerConfig, POCO.O365.EOFolderUpdate folderUpdate) { switch (providerConfig.ProviderType) { case "azure.tableservice": AzureEOFolderUpdate az = new AzureEOFolderUpdate(folderUpdate); CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.EOFolder); TableOperation operation = TableOperation.InsertOrMerge(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); break; case "internal.mongodb": IMongoCollection <MongoEOFolderUpdate> collection = Utils.GetMongoCollection <MongoEOFolderUpdate>(providerConfig, MongoTableNames.EOFolder); MongoEOFolderUpdate mongoObject = Utils.ConvertType <MongoEOFolderUpdate>(folderUpdate); // Create the update filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(mongoObject.PartitionKey), "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(mongoObject.RowKey), "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoEOFolderUpdate> filter = Utils.GenerateMongoFilter <MongoEOFolderUpdate>(filters); var update = Builders <MongoEOFolderUpdate> .Update .Set("TimeCreated", folderUpdate.TimeCreated) .Set("TimeLastModified", folderUpdate.TimeLastModified) .Set("ItemCount", folderUpdate.ItemCount) .Set("Name", folderUpdate.Name) .Set("CPFolderStatus", folderUpdate.CPFolderStatus); // Update the batch status UpdateResult result = collection.UpdateOne(filter, update); return; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return; }
public static string AddLastAction(DataConfig providerConfig, POCO.LogLastAction lastAction) { switch (providerConfig.ProviderType) { case "azure.tableservice": AzureLastAction az = new AzureLastAction(lastAction); CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.LogLastAction); TableOperation operation = TableOperation.InsertOrReplace(az); Task tUpdate = table.ExecuteAsync(operation); tUpdate.Wait(); break; case "internal.mongodb": IMongoCollection <MongoLastAction> collection = Utils.GetMongoCollection <MongoLastAction>(providerConfig, MongoTableNames.LogLastAction); MongoLastAction mongoObject = Utils.ConvertType <MongoLastAction>(lastAction); // Create the upsert filter List <DataFactory.Filter> filters = new List <DataFactory.Filter>(); DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey("LastAction"), "eq"); DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(lastAction.EventName), "eq"); filters.Add(pkFilter); filters.Add(rkFilter); FilterDefinition <MongoLastAction> filter = Utils.GenerateMongoFilter <MongoLastAction>(filters); // Create the upsert options MongoDB.Driver.ReplaceOptions options = new ReplaceOptions(); options.IsUpsert = true; // Upsert string debugfilter = Newtonsoft.Json.JsonConvert.SerializeObject(filters); string debugobject = Newtonsoft.Json.JsonConvert.SerializeObject(mongoObject); collection.ReplaceOne(filter, mongoObject, options); return(string.Empty); default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } //TODO return id of new object if supported return(string.Empty); }