public static void UpdateSiteStatus(DataFactory.DataConfig providerConfig, POCO.System system, POCO.SakaiSite site, string cpStatus)
{
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
throw new NotImplementedException();
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <SakaiSiteEntity>(providerConfig, MongoTableNames.SakaiSites);
// Create the update filter
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq");
DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(site.SITE_ID), "eq");
filters.Add(pkFilter);
filters.Add(rkFilter);
FilterDefinition <SakaiSiteEntity> filter = Utils.GenerateMongoFilter <SakaiSiteEntity>(filters);
string updateParam = "{$set: {TimeCreated: '" + site.CREATEDON.ToUniversalTime().ToString(Utils.ISODateFormat) + "', TimeLastModified: '" + site.MODIFIEDON.ToUniversalTime().ToString(Utils.ISODateFormat) + "', CPStatus: '" + cpStatus + "'}}";
BsonDocument updateDoc = BsonDocument.Parse(updateParam);
// Update the batch status
collection.UpdateOne(filter, updateDoc);
break;
}
}
public static string Add(DataConfig providerConfig, POCO.System system)
{
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
AzureSystem az = new AzureSystem(system);
CloudTable table = Utils.GetCloudTable(providerConfig, TableNames.System);
TableOperation operation = TableOperation.InsertOrMerge(az);
Task tUpdate = table.ExecuteAsync(operation);
tUpdate.Wait();
//TODO return the inserted record id/timestamp
return(string.Empty);
case "internal.mongodb":
IMongoCollection <MongoSystem> collection = Utils.GetMongoCollection <MongoSystem>(providerConfig, TableNames.System);
MongoSystem mongoObject = Utils.ConvertType <MongoSystem>(system);
collection.InsertOne(mongoObject);
return(string.Empty);
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(string.Empty);
}
public IActionResult GetBySystemFilter([FromQuery] string filter)
{
string entityAsJson = "";
try
{
_logger.LogInformation("CPAPI: GetAssigned");
// Deserialize the ontology filter
RecordAuthorityFilter oFilter = new RecordAuthorityFilter();
if (filter != null && filter.Length > 0)
{
_logger.LogDebug("Deserializing filter of length: " + filter.Length);
oFilter = JsonConvert.DeserializeObject <RecordAuthorityFilter>(filter);
}
if (oFilter.systems == null || oFilter.systems.Count == 0)
{
_logger.LogWarning("Missing System filters for GetBySystemFilter");
return(StatusCode((int)System.Net.HttpStatusCode.BadRequest));
}
List <POCO.RecordAuthorityFilter> recauth = new List <POCO.RecordAuthorityFilter>();
DataFactory.DataConfig datacfg = Utils.GetDataConfig();
if (oFilter.systems.Count > 0)
{
foreach (SystemFilter sf in oFilter.systems)
{
POCO.System sys = new POCO.System();
sys.PartitionKey = Utils.CleanTableKey(sf.systemuri);
List <POCO.RecordAuthorityFilter> sysrecauth = DataFactory.RecordAuthority.GetAssignedForSystem(datacfg, sys);
recauth.AddRange(sysrecauth);
}
}
recauth.Sort((x, y) => String.Compare(x.Function, y.Function));
entityAsJson = JsonConvert.SerializeObject(recauth, Formatting.Indented);
}
catch (Exception ex)
{
string exceptionMsg = "Record Authority GET By Filter exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
public AzureSystemStatUpdate(POCO.System system, POCO.SystemStat stat)
{
string jsonstat = JsonConvert.SerializeObject(stat);
this.PartitionKey = system.PartitionKey;
this.RowKey = system.RowKey;
this.JsonSystemStats = jsonstat;
}
public AzureSentenceStatUpdate(POCO.System system, List <POCO.SentenceStat> stats)
{
string jsonstat = JsonConvert.SerializeObject(stats);
this.PartitionKey = system.PartitionKey;
this.RowKey = system.RowKey;
this.JsonSentenceStats = jsonstat;
}
public static void DeleteRegexeMatchs(DataConfig providerConfig, POCO.System system, string fileUri)
{
List <Filter> filters = new List <Filter>();
Filter pk = new Filter("PartitionKey", system.PartitionKey, "eq");
filters.Add(pk);
string rowkey = fileUri;
Filter rk = new Filter("RowKey", rowkey, "ge");
filters.Add(rk);
rk = new Filter("RowKey", Utils.GetLessThanFilter(rowkey), "lt");
filters.Add(rk);
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
// Get a list of entities to delete
List <POCO.CaptureRegexMatch> captureMatches = GetRegexMatches(providerConfig, system, fileUri);
//TODO better way for bulk delete
foreach (POCO.CaptureRegexMatch l in captureMatches)
{
AzureRegexMatch az = new AzureRegexMatch(l);
az.ETag = "*";
CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.RecordAssociationRegexMatch);
TableOperation operation = TableOperation.Delete(az);
Task <TableResult> tDelete = table.ExecuteAsync(operation);
tDelete.Wait();
// Check for "success with no status" code
if (tDelete.Result.HttpStatusCode != 204)
{
// TODO
bool isNotDeleted = true;
}
}
break;
case "internal.mongodb":
FilterDefinition <BsonDocument> filter = Utils.GenerateMongoFilter <BsonDocument>(filters);
// Delete the rows
IMongoCollection <BsonDocument> collection = Utils.GetMongoCollection <BsonDocument>(providerConfig, Record.MongoTableNames.RecordKeyPhrases);
DeleteResult result = collection.DeleteMany(filter);
return;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return;
}
public static string AddProject(DataConfig providerConfig, POCO.System system, POCO.BasecampEntity sourceProject)
{
POCO.BasecampProjectEntity projectEntity = new BasecampProjectEntity(Utils.CleanTableKey(system.PartitionKey), Utils.CleanTableKey(sourceProject.url));
projectEntity.CPStatus = string.Empty;
projectEntity.ItemCount = 0;
projectEntity.Name = sourceProject.name;
projectEntity.ServerRelativeUrl = sourceProject.url;
projectEntity.TimeCreated = sourceProject.created_at.ToUniversalTime();
projectEntity.TimeLastModified = sourceProject.updated_at.ToUniversalTime();
return(BasecampProject.AddProject(providerConfig, projectEntity));
}
public static string AddSite(DataConfig providerConfig, POCO.System system, POCO.SakaiSite sourceSite)
{
POCO.SakaiSiteEntity siteEntity = new SakaiSiteEntity(Utils.CleanTableKey(system.PartitionKey), Utils.CleanTableKey(sourceSite.SITE_ID));
siteEntity.CPStatus = string.Empty;
siteEntity.ItemCount = 0;
siteEntity.Name = sourceSite.TITLE;
siteEntity.ServerRelativeUrl = sourceSite.SITE_ID;
siteEntity.TimeCreated = sourceSite.CREATEDON.ToUniversalTime();
siteEntity.TimeLastModified = sourceSite.MODIFIEDON.ToUniversalTime();
return(SakaiSite.AddSite(providerConfig, siteEntity));
}
public static bool UpdateRecordConfig(DataConfig providerConfig, POCO.System system, string config)
{
bool isUpdatedOk = false;
POCO.RecordConfigUpdate updateRecordCfg = new RecordConfigUpdate();
updateRecordCfg.PartitionKey = system.PartitionKey;
updateRecordCfg.RowKey = system.RowKey;
updateRecordCfg.JsonRecordIdentificationConfig = config;
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
AzureRecordIdentificationConfig az = new AzureRecordIdentificationConfig(updateRecordCfg);
CloudTable table = Utils.GetCloudTable(providerConfig, System.TableNames.System);
TableOperation operation = TableOperation.InsertOrMerge(az);
Task tUpdate = table.ExecuteAsync(operation);
tUpdate.Wait();
isUpdatedOk = true;
break;
case "internal.mongodb":
IMongoCollection <MongoRecordIdentificationConfig> collection = Utils.GetMongoCollection <MongoRecordIdentificationConfig>(providerConfig, System.TableNames.System);
//MongoSystemStatUpdate mongoObject = Utils.ConvertType<MongoSystemStatUpdate>(systemStat);
// Create the update filter
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", updateRecordCfg.PartitionKey, "eq");
DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", updateRecordCfg.RowKey, "eq");
filters.Add(pkFilter);
filters.Add(rkFilter);
FilterDefinition <MongoRecordIdentificationConfig> filter = Utils.GenerateMongoFilter <MongoRecordIdentificationConfig>(filters);
var update = Builders <MongoRecordIdentificationConfig> .Update
.Set("JsonRecordIdentificationConfig", updateRecordCfg.JsonRecordIdentificationConfig);
// Update the batch status
UpdateResult result = collection.UpdateOne(filter, update);
isUpdatedOk = true;
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(isUpdatedOk);
}
public static string AddFileProcessingException(DataConfig providerConfig, POCO.System system, POCO.FileProcessException fileException)
{
string tableName = string.Empty;
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
// Calcuate table name based on System Id and file process result (partitionkey)
tableName = "stlpfileexception-" + fileException.PartitionKey + "-" + system.SystemId.ToString();
AzureFileProcessException az = new AzureFileProcessException(fileException);
CloudTable table = Utils.GetCloudTable(providerConfig, tableName);
TableOperation operation = TableOperation.InsertOrReplace(az);
Task tUpdate = table.ExecuteAsync(operation);
tUpdate.Wait();
break;
case "internal.mongodb":
// Calcuate table name based on System Id and file process result (partitionkey)
tableName = "fileexception-" + fileException.PartitionKey + "-" + system.SystemId.ToString();
IMongoCollection <MongoFileProcessException> collection = Utils.GetMongoCollection <MongoFileProcessException>(providerConfig, tableName);
MongoFileProcessException mongoObject = Utils.ConvertType <MongoFileProcessException>(fileException);
// Create the upsert filter
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", fileException.PartitionKey, "eq");
DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", fileException.RowKey, "eq");
filters.Add(pkFilter);
filters.Add(rkFilter);
FilterDefinition <MongoFileProcessException> filter = Utils.GenerateMongoFilter <MongoFileProcessException>(filters);
// Create the upsert options
MongoDB.Driver.ReplaceOptions options = new ReplaceOptions();
options.IsUpsert = true;
// Upsert
collection.ReplaceOne(filter, mongoObject, options);
return(string.Empty);
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
//TODO return id of new object if supported
return(string.Empty);
}
public static string AddMessage(DataConfig providerConfig, POCO.System system, POCO.GFIMailbox mailbox, POCO.GFIMessage message)
{
string partitionKey = system.PartitionKey;
POCO.GFIMessageEntity docEntity = new GFIMessageEntity(Utils.CleanTableKey(partitionKey), Utils.CleanTableKey(message.MarcId.ToString()));
docEntity.CPStatus = string.Empty;
docEntity.ItemCount = 0;
docEntity.Name = message.Subject;
docEntity.ServerRelativeUrl = string.Empty;
docEntity.TimeCreated = message.SentDate.ToUniversalTime();
docEntity.TimeLastModified = message.ReceivedDate.ToUniversalTime();
return(GFIArchiverMailbox.AddMessage(providerConfig, docEntity));
}
public static string AddFile(DataConfig providerConfig, POCO.System system, POCO.BasecampEntity sourceProject, POCO.BasecampDocument sourceDocument)
{
string partitionKey = GetBasecampBucketUrl(sourceProject.url);
POCO.BasecampDocumentEntity docEntity = new BasecampDocumentEntity(Utils.CleanTableKey(partitionKey), Utils.CleanTableKey(sourceDocument.url));
docEntity.CPStatus = string.Empty;
docEntity.ItemCount = 0;
docEntity.Name = sourceDocument.title;
docEntity.ServerRelativeUrl = sourceDocument.url;
docEntity.TimeCreated = sourceDocument.created_at.ToUniversalTime();
docEntity.TimeLastModified = sourceDocument.updated_at.ToUniversalTime();
docEntity.Title = sourceDocument.title;
return(BasecampProject.AddFile(providerConfig, docEntity));
}
public static void UpdateSentenceStat(DataConfig providerConfig, POCO.System system, List <POCO.SentenceStat> sentenceStats)
{
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
AzureSentenceStatUpdate az = new AzureSentenceStatUpdate(system, sentenceStats);
CloudTable table = Utils.GetCloudTable(providerConfig, System.TableNames.System);
TableOperation operation = TableOperation.InsertOrMerge(az);
Task tUpdate = table.ExecuteAsync(operation);
tUpdate.Wait();
break;
case "internal.mongodb":
IMongoCollection <MongoSystemStatUpdate> collection = Utils.GetMongoCollection <MongoSystemStatUpdate>(providerConfig, "stlpsystems");
//MongoSystemStatUpdate mongoObject = Utils.ConvertType<MongoSystemStatUpdate>(systemStat);
// Create the update filter
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter pkFilter = new DataFactory.Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq");
//DataFactory.Filter rkFilter = new DataFactory.Filter("RowKey", Utils.CleanTableKey(system.RowKey), "eq");
filters.Add(pkFilter);
//filters.Add(rkFilter);
FilterDefinition <MongoSystemStatUpdate> filter = Utils.GenerateMongoFilter <MongoSystemStatUpdate>(filters);
// Serialize the stats object
string jsonStatsSerialized = JsonConvert.SerializeObject(sentenceStats);
//string updateParam = "{$set: {JsonSentenceStats: '" + jsonStatsSerialized + "'}}";
//BsonDocument updateDoc = BsonDocument.Parse(updateParam);
var update = Builders <MongoSystemStatUpdate> .Update
.Set("JsonSentenceStats", jsonStatsSerialized);
// Update the batch status
UpdateResult result = collection.UpdateOne(filter, update);
return;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return;
}
public static string AddFile(DataConfig providerConfig, POCO.System system, POCO.SakaiSite sourceSite, POCO.SakaiContentResource sourceDocument)
{
// Set the PartitionKey and RowKey
string sakaiUIDocument = "/" + sourceSite.SITE_ID + sourceDocument.RESOURCE_ID;
string pkey = Utils.CleanTableKey(sakaiUIDocument);
string rkey = Utils.CleanTableKey(sourceDocument.TimeLastModified.ToString(Utils.ISODateFormat));
// Get the file name from the file path
string fileName = sourceDocument.RESOURCE_ID.Substring(sourceDocument.RESOURCE_ID.LastIndexOf("/") + 1);
POCO.SakaiFile sakaifile = new SakaiFile(pkey, rkey);
//sakaifile.BatchGuid = string.Empty;
sakaifile.BatchStatus = string.Empty;
sakaifile.CPFolderStatus = string.Empty;
sakaifile.CreationTime = sourceDocument.TimeCreated;
sakaifile.ItemCount = 0;
sakaifile.ItemUri = sakaiUIDocument;
sakaifile.LastModifiedTime = sourceDocument.TimeLastModified;
sakaifile.Name = fileName;
sakaifile.ServerRelativeUrl = sourceDocument.FILE_PATH;
sakaifile.SizeInBytes = sourceDocument.FILE_SIZE;
sakaifile.SourceFileName = fileName;
sakaifile.SourceRelativeUrl = sakaiUIDocument;
sakaifile.UniqueId = sourceDocument.RESOURCE_UUID;
sakaifile.Version = 0;
//POCO.SakaiDocumentEntity docEntity = new SakaiDocumentEntity(Utils.CleanTableKey(sourceSite.SITE_ID), Utils.CleanTableKey(sourceDocument.FILE_PATH));
//docEntity.CPStatus = string.Empty;
//docEntity.UniqueId = sourceDocument.RESOURCE_ID;
//docEntity.ItemCount = 0;
//docEntity.Name = fileName;
//docEntity.ServerRelativeUrl = sourceDocument.FILE_PATH;
//docEntity.TimeCreated = sourceDocument.TimeCreated;
//docEntity.TimeLastModified = sourceDocument.TimeLastModified;
//docEntity.Title = fileName;
return(SakaiSite.AddFile(providerConfig, sakaifile));
}
public static string GetMIMEType(DataConfig providerConfig, POCO.System system, POCO.Record record, RecordToRecordAssociation recordAssocEntity, ILogger logger)
{
string mimeType = string.Empty;
List <DataFactory.Filter> filters = new List <Filter>();
DataFactory.Filter pkFilter = new Filter("PartitionKey", recordAssocEntity.RowKey, "eq");
filters.Add(pkFilter);
//DataFactory.Filter rkFilter = new Filter("RowKey", , "eq");
//filters.Add(rkFilter);
string nextPageId = string.Empty;
string tableName = string.Empty;
// Check which System the file is from
switch (system.Type)
{
case SystemType.SharePoint2010:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePoint.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePoint.SPFile;
break;
}
break;
case SystemType.SharePoint2013:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePoint.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePoint.SPFile;
break;
}
break;
case SystemType.SharePointOnline:
case SystemType.SharePointOneDrive:
case SystemType.SharePointTeam:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePointOnline.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePointOnline.SPFile;
break;
}
break;
case SystemType.NTFSShare:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.NTFS.NTFSFiles;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.NTFS.NTFSFiles;
break;
}
break;
case SystemType.SakaiAlliance:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.Sakai.SakaiFiles;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.Sakai.SakaiFiles;
break;
}
break;
default:
throw new NotImplementedException();
break;
}
List <POCO.CPFileMIMEType> mimetypes = new List <POCO.CPFileMIMEType>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureFileMIMEType> azdata = new List <AzureFileMIMEType>();
AzureTableAdaptor <AzureFileMIMEType> adaptor = new AzureTableAdaptor <AzureFileMIMEType>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
mimetypes.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoFileMIMEType>(providerConfig, tableName);
FilterDefinition <MongoFileMIMEType> filter = Utils.GenerateMongoFilter <MongoFileMIMEType>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
mimetypes.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
if (mimetypes.Count > 0)
{
mimeType = mimetypes[0].MIMEType;
}
return(mimeType);
}
public static List <POCO.SakaiSiteEntity> GetSites(DataConfig providerConfig, POCO.System system, POCO.SakaiSite site)
{
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq");
filters.Add(pkFilter);
Filter rkFilter = new Filter("RowKey", Utils.CleanTableKey(site.SITE_ID), "eq");
filters.Add(rkFilter);
return(SakaiSite.GetSites(providerConfig, filters));
}
public static List <POCO.BasecampProjectEntity> GetProjects(DataConfig providerConfig, POCO.System system, POCO.BasecampEntity basecampProject)
{
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq");
Filter rkFilter = new Filter("RowKey", Utils.CleanTableKey(basecampProject.url), "eq");
filters.Add(pkFilter);
filters.Add(rkFilter);
return(BasecampProject.GetProjects(providerConfig, filters));
}
public static List <POCO.RecordAuthorityFilter> GetAssignedForSystem(DataConfig providerConfig, POCO.System system)
{
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", system.PartitionKey, "eq");
filters.Add(pkFilter);
return(GetAssignedForSystem(providerConfig, filters));
}
public static long DeleteFileProcessStats(DataConfig providerConfig, POCO.System system)
{
long numEntriesDeleted = 0;
string systemKey = system.PartitionKey;
if (!systemKey.EndsWith("|"))
{
systemKey += "|";
}
// Create a filter for the system
List <Filter> filters = new List <Filter>();
Filter pkfilt = new Filter("PartitionKey", systemKey, "eq");
filters.Add(pkfilt);
// Check the data provider to use
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.FileProcessStats);
TableQuery <TableEntity> query = new TableQuery <TableEntity>().Where(combinedFilter);
List <TableEntity> recordsToDelete = new List <TableEntity>();
TableContinuationToken token = null;
var runningQuery = new TableQuery <TableEntity>()
{
FilterString = query.FilterString,
SelectColumns = query.SelectColumns
};
do
{
runningQuery.TakeCount = query.TakeCount - recordsToDelete.Count;
// Add the entries to the recordsToDelete list
Task <TableQuerySegment <TableEntity> > tSeg = table.ExecuteQuerySegmentedAsync <TableEntity>(runningQuery, token);
tSeg.Wait();
token = tSeg.Result.ContinuationToken;
recordsToDelete.AddRange(tSeg.Result);
// Create a batch of records to delete
TableBatchOperation batch = new TableBatchOperation();
foreach (TableEntity entity in recordsToDelete)
{
batch.Add(TableOperation.Delete(entity));
if (batch.Count == 100)
{
numEntriesDeleted += batch.Count;
Task tBatchDelete = table.ExecuteBatchAsync(batch);
tBatchDelete.Wait();
batch = new TableBatchOperation();
}
}
if (batch.Count > 0)
{
numEntriesDeleted += batch.Count;
Task tBatchDelete = table.ExecuteBatchAsync(batch);
tBatchDelete.Wait();
}
} while (token != null && (query.TakeCount == null || recordsToDelete.Count < query.TakeCount.Value)); //!ct.IsCancellationRequested &&
break;
case "internal.mongodb":
IMongoCollection <BsonDocument> collection = Utils.GetMongoCollection <BsonDocument>(providerConfig, MongoTableNames.FileProcessStats);
// Create the delete filter
FilterDefinition <BsonDocument> filter = Utils.GenerateMongoFilter <BsonDocument>(filters);
// Replace current document
DeleteResult result = collection.DeleteMany(filter);
numEntriesDeleted = result.DeletedCount;
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(numEntriesDeleted);
}
public static List <POCO.Files.CPFile> GetFiles(DataConfig dataConfig, POCO.System system, List <Filter> filters, string thisPageId, int maxRows, out string nextPageId)
{
List <POCO.Files.CPFile> filesToProcess = new List <POCO.Files.CPFile>();
// Default next page id to "no more data" (empty string)
nextPageId = string.Empty;
// Check the system type
switch (system.Type)
{
case POCO.SystemType.SharePoint2010:
{
List <POCO.SharePoint.SPFile> spfiles = DataFactory.SharePoint.GetFiles(dataConfig, filters, thisPageId, maxRows, out nextPageId);
// Convert to ICPFile format
foreach (POCO.SharePoint.SPFile fileToConvert in spfiles)
{
// Convert to ICPFile compatible object
POCO.Files.SPFile file = JsonConvert.DeserializeObject <POCO.Files.SPFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
break;
}
case POCO.SystemType.SharePoint2013:
{
List <POCO.SharePoint.SPFile> spfiles = DataFactory.SharePoint.GetFiles(dataConfig, filters, thisPageId, maxRows, out nextPageId);
// Convert to ICPFile format
foreach (POCO.SharePoint.SPFile fileToConvert in spfiles)
{
// Convert to ICPFile compatible object
POCO.Files.SPFile file = JsonConvert.DeserializeObject <POCO.Files.SPFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
break;
}
case POCO.SystemType.SharePointOneDrive:
case POCO.SystemType.SharePointTeam:
case POCO.SystemType.SharePointOnline:
{
List <POCO.SPFile> spfiles = DataFactory.SharePointOnline.GetFiles(dataConfig, filters, thisPageId, maxRows, out nextPageId);
// Convert to ICPFile format
foreach (POCO.SPFile fileToConvert in spfiles)
{
// Convert to ICPFile compatible object
POCO.Files.SPFile file = JsonConvert.DeserializeObject <POCO.Files.SPFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
break;
}
case POCO.SystemType.GoogleDrive:
{
List <POCO.Files.GoogleFile> googlefiles = DataFactory.Google.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.Files.GoogleFile fileToConvert in googlefiles)
{
// Convert to ICPFile compatible object
POCO.Files.GoogleFile file = JsonConvert.DeserializeObject <POCO.Files.GoogleFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
break;
}
case POCO.SystemType.NTFSShare:
List <POCO.NTFSFile> ntfsFiles = DataFactory.NTFS.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.NTFSFile fileToConvert in ntfsFiles)
{
// Convert to ICPFile compatible object
POCO.Files.NTFSFile ntfsfile = JsonConvert.DeserializeObject <POCO.Files.NTFSFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(ntfsfile);
}
break;
case POCO.SystemType.SakaiAlliance:
List <POCO.SakaiFile> sakaiFiles = DataFactory.SakaiResource.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.SakaiFile fileToConvert in sakaiFiles)
{
// Convert to ICPFile compatible object
POCO.Files.SakaiFile sakaifile = JsonConvert.DeserializeObject <POCO.Files.SakaiFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(sakaifile);
}
break;
case POCO.SystemType.Basecamp:
// Get the Basecamp files
throw new NotImplementedException("GetFiles: system type not implemented=" + system.Type);
// URL format of the basecamp project is: https://<basecamp api>/<account number>/projects/<project id>.json
// all data for a project is in "buckets": https://<basecamp api>/<account number>/buckets/<project id>/<type of bucket>/<id>.json
// any data matching should use the buckets/<project id>>
//string basecampFilesUri = record.RecordUri.Replace("projects", "buckets");
//if (basecampFilesUri.ToLower().EndsWith(".json")) { basecampFilesUri = basecampFilesUri.Substring(0, basecampFilesUri.Length - 5); }
//if (!basecampFilesUri.EndsWith("|")) { basecampFilesUri += "|"; }
//List<DataFactory.Filter> basecampFilters = new List<DataFactory.Filter>();
//string pk = Utils.CleanTableKey(basecampFilesUri);
//DataFactory.Filter basecampPK = new DataFactory.Filter("PartitionKey", pk, "eq");
//basecampFilters.Add(basecampPK);
////DataFactory.Filter basecampPKLT = new DataFactory.Filter("PartitionKey", Utils.GetLessThanFilter(pk), "lt");
////basecampFilters.Add(basecampPKGE);
////basecampFilters.Add(basecampPKLT);
//DataFactory.DataConfig basecampDataConfig = dataConfig;
//List<POCO.BasecampDocumentEntity> basecampFiles = DataFactory.BasecampProject.GetFiles(basecampDataConfig, basecampFilters);
//// Convert to ICPFile format
//foreach (POCO.BasecampDocumentEntity fileToConvert in basecampFiles)
//{
// // Convert to ICPFile compatible object
// //BasecampFile basefile = JsonConvert.DeserializeObject<BasecampFile>(JsonConvert.SerializeObject(fileToConvert));
// POCO.Files.BasecampFile bf = new POCO.Files.BasecampFile();
// bf.BatchGuid = Guid.Empty;
// bf.BatchStatus = null;
// bf.CreationTime = fileToConvert.TimeCreated;
// bf.FolderUri = fileToConvert.PartitionKey;
// bf.ItemUri = fileToConvert.ServerRelativeUrl;
// bf.JsonFileProcessResult = null;
// bf.LastModifiedTime = fileToConvert.TimeLastModified;
// bf.Name = fileToConvert.Name;
// bf.OrganisationId = Guid.Empty;
// bf.PartitionKey = fileToConvert.PartitionKey;
// bf.RowKey = fileToConvert.RowKey;
// bf.SiteUrl = fileToConvert.ServerRelativeUrl;
// bf.SourceFileName = fileToConvert.Name;
// bf.SourceRelativeUrl = fileToConvert.ServerRelativeUrl;
// bf.Title = fileToConvert.Title;
// bf.UniqueId = fileToConvert.UniqueId;
// filesToProcess.Add(bf);
//}
break;
default:
{
throw new NotImplementedException("GetFiles: unknown system type: " + system.Type);
}
}
return(filesToProcess);
}
internal static List <POCO.Files.CPFile> GetFilesAssociatedToRecordForBatchProcessing(DataConfig dataConfig, POCO.System system, POCO.Record record)
{
List <POCO.Files.CPFile> filesToProcess = new List <POCO.Files.CPFile>();
// Create the filter
string recordUri = record.RowKey;
if (!recordUri.EndsWith("|"))
{
recordUri += "|";
} // Standardise the url to end in a pipe '|' to help match files via partial Uri
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter pkGTEFilter = new DataFactory.Filter("PartitionKey", recordUri, "ge");
DataFactory.Filter pkLTFilter = new DataFactory.Filter("PartitionKey", Utils.GetLessThanFilter(recordUri), "lt");
filters.Add(pkGTEFilter);
filters.Add(pkLTFilter);
// Check the system type
string thisPageId = string.Empty;
string nextPageId = string.Empty;
switch (system.Type)
{
case POCO.SystemType.SharePointOnline:
case POCO.SystemType.SharePointTeam:
case POCO.SystemType.SharePointOneDrive:
{
do
{
List <POCO.SPFile> spfiles = DataFactory.SharePointOnline.GetFiles(dataConfig, filters, thisPageId, 1000, out nextPageId);
// Convert to ICPFile format
foreach (POCO.SPFile fileToConvert in spfiles)
{
// Convert to ICPFile compatible object
POCO.Files.SPFile file = JsonConvert.DeserializeObject <POCO.Files.SPFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
thisPageId = nextPageId;
}while (nextPageId != string.Empty);
break;
}
case POCO.SystemType.GoogleDrive:
{
List <POCO.Files.GoogleFile> googlefiles = DataFactory.Google.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.Files.GoogleFile fileToConvert in googlefiles)
{
// Convert to ICPFile compatible object
POCO.Files.GoogleFile file = JsonConvert.DeserializeObject <POCO.Files.GoogleFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
break;
}
case POCO.SystemType.NTFSShare:
List <POCO.NTFSFile> ntfsFiles = DataFactory.NTFS.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.NTFSFile fileToConvert in ntfsFiles)
{
// Convert to ICPFile compatible object
POCO.Files.NTFSFile ntfsfile = JsonConvert.DeserializeObject <POCO.Files.NTFSFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(ntfsfile);
}
break;
case POCO.SystemType.Basecamp:
// Get the Basecamp files
// URL format of the basecamp project is: https://<basecamp api>/<account number>/projects/<project id>.json
// all data for a project is in "buckets": https://<basecamp api>/<account number>/buckets/<project id>/<type of bucket>/<id>.json
// any data matching should use the buckets/<project id>>
string basecampFilesUri = record.RecordUri.Replace("projects", "buckets");
if (basecampFilesUri.ToLower().EndsWith(".json"))
{
basecampFilesUri = basecampFilesUri.Substring(0, basecampFilesUri.Length - 5);
}
if (!basecampFilesUri.EndsWith("|"))
{
basecampFilesUri += "|";
}
List <DataFactory.Filter> basecampFilters = new List <DataFactory.Filter>();
string pk = Utils.CleanTableKey(basecampFilesUri);
DataFactory.Filter basecampPK = new DataFactory.Filter("PartitionKey", pk, "eq");
basecampFilters.Add(basecampPK);
//DataFactory.Filter basecampPKLT = new DataFactory.Filter("PartitionKey", Utils.GetLessThanFilter(pk), "lt");
//basecampFilters.Add(basecampPKGE);
//basecampFilters.Add(basecampPKLT);
DataFactory.DataConfig basecampDataConfig = dataConfig;
List <POCO.BasecampDocumentEntity> basecampFiles = DataFactory.BasecampProject.GetFiles(basecampDataConfig, basecampFilters);
// Convert to ICPFile format
foreach (POCO.BasecampDocumentEntity fileToConvert in basecampFiles)
{
// Convert to ICPFile compatible object
//BasecampFile basefile = JsonConvert.DeserializeObject<BasecampFile>(JsonConvert.SerializeObject(fileToConvert));
POCO.Files.BasecampFile bf = new POCO.Files.BasecampFile();
bf.BatchGuid = Guid.Empty;
bf.BatchStatus = null;
bf.CreationTime = fileToConvert.TimeCreated;
bf.FolderUri = fileToConvert.PartitionKey;
bf.ItemUri = fileToConvert.ServerRelativeUrl;
bf.JsonFileProcessResult = null;
bf.LastModifiedTime = fileToConvert.TimeLastModified;
bf.Name = fileToConvert.Name;
bf.OrganisationId = Guid.Empty;
bf.PartitionKey = fileToConvert.PartitionKey;
bf.RowKey = fileToConvert.RowKey;
bf.SiteUrl = fileToConvert.ServerRelativeUrl;
bf.SourceFileName = fileToConvert.Name;
bf.SourceRelativeUrl = fileToConvert.ServerRelativeUrl;
bf.Title = fileToConvert.Title;
bf.UniqueId = fileToConvert.UniqueId;
filesToProcess.Add(bf);
}
break;
default:
{
throw new NotImplementedException("Unknown system type: " + system.Type);
}
}
return(filesToProcess);
}
public static List <POCO.OntologyAssigned> GetSystemAssignedOntology(DataConfig providerConfig, POCO.System system)
{
List <POCO.OntologyAssigned> ontologies = new List <POCO.OntologyAssigned>();
List <Filter> filters = new List <Filter>();
Filter systemFilter = new Filter("PartitionKey", Utils.CleanTableKey(system.PartitionKey), "eq");
filters.Add(systemFilter);
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureOntologyAssigned> azs = new List <AzureOntologyAssigned>();
AzureTableAdaptor <AzureOntologyAssigned> az = new AzureTableAdaptor <AzureOntologyAssigned>();
azs = az.ReadTableData(providerConfig, AzureTableNames.SystemsAssignedOntology, combinedFilter);
foreach (var doc in azs)
{
ontologies.Add(doc.Value);
}
return(ontologies);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoOntologyAssigned>(providerConfig, MongoTableNames.SystemsAssignedOntology);
FilterDefinition <MongoOntologyAssigned> filter = Utils.GenerateMongoFilter <MongoOntologyAssigned>(filters);
var documents = collection.Find(filter).ToList();
foreach (var ontology in documents)
{
ontologies.Add(ontology);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(ontologies);
}
public static List <POCO.CaptureRegexMatch> GetRegexMatches(DataConfig providerConfig, POCO.System system, string fileUri)
{
// Create a filter to match the Ontology provided
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", system.PartitionKey, "eq");
filters.Add(pkFilter);
string rowkey = fileUri;
Filter rk = new Filter("RowKey", rowkey, "ge");
filters.Add(rk);
rk = new Filter("RowKey", Utils.GetLessThanFilter(rowkey), "lt");
filters.Add(rk);
return(GetRegexMatches(providerConfig, filters));
}
public static List <POCO.FileProcessException> GetFileProcessException(DataConfig providerConfig, POCO.System system, string exceptionType)
{
string tableName = string.Empty;
List <POCO.FileProcessException> docs = new List <POCO.FileProcessException>();
// Create filter
List <Filter> filters = new List <Filter>();
Filter pkfilt = new Filter("PartitionKey", Utilities.Converters.CleanTableKey(exceptionType), "eq");
filters.Add(pkfilt);
switch (providerConfig.ProviderType)
{
case ProviderType.Azure:
// Calcuate table name based on System Id and file process result (partitionkey)
tableName = "stlpfileexception-" + Utilities.Converters.CleanTableKey(exceptionType) + "-" + system.SystemId.ToString();
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureFileProcessException> azdata = new List <AzureFileProcessException>();
AzureTableAdaptor <AzureFileProcessException> adaptor = new AzureTableAdaptor <AzureFileProcessException>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
docs.Add(doc.Value);
}
break;
case ProviderType.Mongo:
// Calcuate table name based on System Id and file process result (partitionkey)
tableName = "fileexception-" + Utilities.Converters.CleanTableKey(exceptionType) + "-" + system.SystemId.ToString();
var collection = Utils.GetMongoCollection <MongoFileProcessException_read>(providerConfig, tableName);
FilterDefinition <MongoFileProcessException_read> filter = Utils.GenerateMongoFilter <MongoFileProcessException_read>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
docs.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(docs);
}
