public FormData GetSubmittedValues(IPublishedContent content, int page = 1, int perPage = 10, string sortField = null,
bool sortDescending = false, ApprovalState approvalState = ApprovalState.Approved, string searchQuery = null, string[] searchFields = null)
{
var index = IndexHelper.GetIndex(content.Id);
IApprovalIndex approvalIndex = null;
if (index is IApprovalIndex)
{
// we need to load the prevalues to figure out if we're using approval or not
LoadPreValues(content);
approvalIndex = UseApproval ? index as IApprovalIndex : null;
}
var result = approvalIndex != null
? approvalIndex.Get(searchQuery, searchFields, sortField, sortDescending, perPage, (page - 1) *perPage, approvalState)
: index.Get(searchQuery, searchFields, sortField, sortDescending, perPage, (page - 1) * perPage);
result = result ?? Result.Empty(sortField, sortDescending);
var fields = AllValueFields().ToArray();
var rows = ExtractSubmittedValues(result, fields, (field, value, row) => value == null ? null : field.FormatValueForFrontend(value, content, row.Id));
return(new FormData
{
TotalRows = result.TotalRows,
SortDescending = result.SortDescending,
SortField = result.SortField,
Rows = rows.ToArray(),
Fields = fields.Select(f => ToDataField(null, f, null))
});
}
public object GetData(int id, int page, string sortField, bool sortDescending, string searchQuery = null)
{
// NOTE: this is fine for now, but eventually make it should probably be configurable
const int PerPage = 10;
var document = ContentHelper.GetById(id);
if (document == null)
{
return(null);
}
var model = ContentHelper.GetFormModel(document);
if (model == null)
{
return(null);
}
var preValues = ContentHelper.GetPreValues(document, FormModel.PropertyEditorAlias);
var allFields = GetAllFieldsForDisplay(model, document, preValues);
var statisticsEnabled = ContentHelper.StatisticsEnabled(preValues);
var approvalEnabled = ContentHelper.ApprovalEnabled(preValues);
var index = IndexHelper.GetIndex(id);
var fullTextIndex = index as IFullTextIndex;
var result = (fullTextIndex != null && string.IsNullOrWhiteSpace(searchQuery) == false
? fullTextIndex.Search(searchQuery, allFields.Select(f => f.FormSafeName).ToArray(), sortField, sortDescending, PerPage, (page - 1) * PerPage)
: index.Get(searchQuery, null, sortField, sortDescending, PerPage, (page - 1) * PerPage)
) ?? Result.Empty(sortField, sortDescending);
var totalPages = (int)Math.Ceiling((double)result.TotalRows / PerPage);
// out of bounds request - e.g. right after removing some rows?
if (page > totalPages && totalPages > 0)
{
// repeat the query but get the last page
page = totalPages;
result = index.Get(searchQuery, null, sortField, sortDescending, PerPage, (page - 1) * PerPage);
}
var rows = model.ExtractSubmittedValues(result, allFields, (field, value, row) => field.FormatValueForDataView(value, document, row.Id));
return(new
{
fields = allFields.Select(f => new { name = f.Name, sortName = f.FormSafeName }).ToArray(),
rows = rows.Select(r => new
{
_id = r.Id,
_createdDate = r.CreatedDate,
_approval = r.ApprovalState.ToString().ToLowerInvariant(),
values = r.Fields.Select(f => f.Value)
}).ToArray(),
currentPage = page,
totalPages = totalPages,
sortField = result.SortField,
sortDescending = result.SortDescending,
supportsSearch = fullTextIndex != null,
supportsStatistics = statisticsEnabled && index is IStatisticsIndex && allFields.StatisticsFields().Any(),
supportsApproval = approvalEnabled && index is IApprovalIndex
});
}
public HttpResponseMessage SearchAll(string searchQuery)
{
var contentResults = new List <ContentResult>();
ContentHelper.ForEachFormModel(ApplicationContext.Services, (formModel, content) =>
{
if (!(IndexHelper.GetIndex(content.Id) is IFullTextIndex index))
{
return;
}
var fields = formModel.AllValueFields().ToArray();
var result = index.Search(searchQuery, fields.Select(f => f.FormSafeName).ToArray(), null, false, 100, 0);
if (result == null || result.TotalRows == 0)
{
return;
}
var rows = formModel.ExtractSubmittedValues(result, fields, (field, value, row) => value == null ? null : field.FormatValueForDataView(value, content, row.Id));
contentResults.Add(new ContentResult
{
ContentId = content.Id,
ContentName = content.Name,
FieldNames = fields.Select(f => f.Name).ToArray(),
Rows = rows.ToArray()
});
});
private Guid AddSubmittedValuesToIndex(IPublishedContent content, IEnumerable <FieldWithValue> valueFields)
{
var rowId = Guid.NewGuid();
// extract all index values
var indexFields = valueFields.ToDictionary(f => f.FormSafeName, f => FormatForIndexAndSanitize(f, content, rowId));
// add the IP of the user if enabled on the data type
if (LogIp)
{
indexFields.Add("_ip", Request.UserHostAddress);
}
// store fields in index
var index = IndexHelper.GetIndex(content.Id);
var statisticsIndex = index as IStatisticsIndex;
if (UseStatistics && statisticsIndex != null)
{
var indexFieldsForStatistics = valueFields.StatisticsFields().ToDictionary(f => f.FormSafeName, f => f.SubmittedValues ?? new string[] {});
statisticsIndex.Add(indexFields, indexFieldsForStatistics, rowId);
}
else
{
index.Add(indexFields, rowId);
}
return(rowId);
}
protected internal override string FormatSubmittedValueForIndex(IPublishedContent content, Guid rowId)
{
var file = GetUploadedFile();
if (file == null)
{
return(null);
}
var indexValue = ParseIndexValue(SubmittedValue);
if (indexValue == null)
{
return(null);
}
// save the file to the index
var index = IndexHelper.GetIndex(content.Id);
if (index.SaveFile(file, indexValue.PersistedFilename, rowId) == false)
{
return(null);
}
return(SubmittedValue);
}
public HttpResponseMessage DownloadFile(int id, Guid rowId, string fieldName)
{
var index = IndexHelper.GetIndex(id);
var row = index.Get(rowId);
if (row == null || row.Fields.ContainsKey(fieldName) == false)
{
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var indexValue = row.Fields[fieldName];
var fileData = UploadField.ParseIndexValue(indexValue);
if (fileData == null)
{
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var stream = index.GetFile(fileData.PersistedFilename);
if (stream == null)
{
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var response = new HttpResponseMessage(HttpStatusCode.OK);
response.Content = new StreamContent(stream);
response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = fileData.OriginalFilename
};
return(response);
}
public void LoadValues(IPublishedContent content, Guid rowId)
{
if (rowId == Guid.Empty)
{
return;
}
var index = IndexHelper.GetIndex(content.Id);
var formData = index.Get(rowId);
if (formData == null)
{
return;
}
RowId = rowId;
var fields = AllFields().ToArray();
foreach (var field in fields)
{
field.CollectSubmittedValue(formData.Fields, content);
}
foreach (var field in fields)
{
// using ValidateSubmittedValue to load up select boxes
field.ValidateSubmittedValue(fields, content);
}
}
public HttpResponseMessage PurgeExpiredSubmissions(string authToken)
{
// validate authentication token
if (FormEditor.Configuration.Instance.Jobs == null || FormEditor.Configuration.Instance.Jobs.IsValidAuthToken(authToken) == false)
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Invalid authentication token"));
}
try
{
ContentHelper.ForEachFormModel(ApplicationContext.Services, (formModel, content) =>
{
if (formModel.DaysBeforeSubmissionExpiry.HasValue == false || formModel.DaysBeforeSubmissionExpiry.Value <= 0)
{
return;
}
var olderThan = DateTime.UtcNow.AddDays(-1 * formModel.DaysBeforeSubmissionExpiry.Value);
if (!(IndexHelper.GetIndex(content.Id) is IAutomationIndex index))
{
Log.Warning($"Unable to purge expired submissions - the configured storage index is not of type {nameof(IAutomationIndex)}");
return;
}
index.RemoveOlderThan(olderThan);
});
return(Request.CreateResponse(HttpStatusCode.OK));
}
public bool MaxSubmissionsExceeded(IPublishedContent content)
{
if (MaxSubmissions.HasValue == false)
{
return(false);
}
var index = IndexHelper.GetIndex(content.Id);
return(index.Count() >= MaxSubmissions.Value);
}
public object SetApprovalState(int id, SetApprovalStateRequest request)
{
if (IndexHelper.GetIndex(id) is IApprovalIndex approvalIndex && approvalIndex.SetApprovalState(request.ApprovalState, request.RowId))
{
return(new
{
newApprovalState = request.ApprovalState.ToString().ToLowerInvariant()
});
}
return(null);
}
public object GetFieldValueFrequencyStatistics(int id)
{
var document = ContentHelper.GetById(id);
if (document == null)
{
return(null);
}
var model = ContentHelper.GetFormModel(document);
if (model == null)
{
return(null);
}
var statisticsFields = model.AllValueFields().OfType <IValueFrequencyStatisticsField>().ToList();
if (statisticsFields.Any() == false)
{
return(null);
}
var index = IndexHelper.GetIndex(id) as IStatisticsIndex;
if (index == null)
{
return(null);
}
var fieldValueFrequencyStatistics = index.GetFieldValueFrequencyStatistics(statisticsFields.StatisticsFieldNames());
return(new
{
totalRows = fieldValueFrequencyStatistics.TotalRows,
fields = fieldValueFrequencyStatistics.FieldValueFrequencies
.Where(f => statisticsFields.Any(v => v.FormSafeName == f.Field))
.Select(f =>
{
var field = statisticsFields.First(v => v.FormSafeName == f.Field);
return new
{
name = field.Name,
formSafeName = field.FormSafeName,
multipleValuesPerEntry = field.MultipleValuesPerEntry,
values = f.Frequencies.Select(v => new
{
value = v.Value,
frequency = v.Frequency
})
};
})
});
}
public object GetData(int id, int page, string sortField, bool sortDescending)
{
// NOTE: this is fine for now, but eventually make it should probably be configurable
const int PerPage = 10;
var document = ContentHelper.GetById(id);
if (document == null)
{
return(null);
}
var model = ContentHelper.GetFormModel(document);
if (model == null)
{
return(null);
}
var allFields = GetAllFieldsForDisplay(model, document);
var index = IndexHelper.GetIndex(id);
var result = index.Get(sortField, sortDescending, PerPage, (page - 1) * PerPage);
var totalPages = (int)Math.Ceiling((double)result.TotalRows / PerPage);
// out of bounds request - e.g. right after removing some rows?
if (page > totalPages && totalPages > 0)
{
// repeat the query but get the last page
page = totalPages;
result = index.Get(sortField, sortDescending, PerPage, (page - 1) * PerPage);
}
var rows = model.ExtractSubmittedValues(result, allFields, (field, value, row) => field.FormatValueForDataView(value, document, row.Id));
return(new
{
fields = allFields.Select(f => new { name = f.Name, sortName = f.FormSafeName }).ToArray(),
rows = rows.Select(r => new
{
_id = r.Id,
_createdDate = r.CreatedDate,
values = r.Fields.Select(f => f.Value)
}).ToArray(),
currentPage = page,
totalPages = totalPages,
sortField = result.SortField,
sortDescending = result.SortDescending,
});
}
private void AddSubmittedValuesToIndex(IPublishedContent content, IEnumerable <FieldWithValue> valueFields)
{
// extract all index values
var indexFields = valueFields.ToDictionary(f => f.FormSafeName, f => FormatForIndexAndSanitize(f, content));
// add the IP of the user if enabled on the data type
if (LogIp)
{
indexFields.Add("_ip", HttpContext.Current.Request.UserHostAddress);
}
// store fields in index
var index = IndexHelper.GetIndex(content.Id);
index.Add(indexFields);
}
private Guid AddSubmittedValuesToIndex(IPublishedContent content, IEnumerable <FieldWithValue> valueFields)
{
// we're performing an update if RowId as a value
var isUpdate = RowId != Guid.Empty;
// generate a new row ID for the index only if we're not performing an update, otherwise reuse RowId
var indexRowId = isUpdate ? RowId : Guid.NewGuid();
// get the storage index
var index = IndexHelper.GetIndex(content.Id);
// - attempt to cast to IStatisticsIndex if statistics are enabled
var statisticsIndex = UseStatistics ? index as IStatisticsIndex : null;
// - attempt to cast to IUpdateIndex if we're performing an update
// (this will change in an upcoming release when IUpdateIndex is merged into IIndex)
var updateIndex = isUpdate ? index as IUpdateIndex : null;
if (isUpdate)
{
// can we perform the update? only IStatisticsIndex and IUpdateIndex support updates
if (statisticsIndex == null && updateIndex == null)
{
return(Guid.Empty);
}
}
// extract all index values
var indexFields = valueFields.ToDictionary(f => f.FormSafeName, f => FormatForIndexAndSanitize(f, content, indexRowId));
// add the IP of the user if enabled on the data type
if (LogIp)
{
indexFields.Add("_ip", Request.UserHostAddress);
}
if (statisticsIndex != null)
{
var indexFieldsForStatistics = valueFields.StatisticsFields().ToDictionary(f => f.FormSafeName, f => f.SubmittedValues ?? new string[] {});
return(isUpdate
? statisticsIndex.Update(indexFields, indexFieldsForStatistics, indexRowId)
: statisticsIndex.Add(indexFields, indexFieldsForStatistics, indexRowId));
}
if (updateIndex != null)
{
return(updateIndex.Update(indexFields, indexRowId));
}
return(index.Add(indexFields, indexRowId));
}
private static void DeleteEntityIndex(IContent deletedEntity)
{
try
{
var formModelProperty = ContentHelper.GetFormModelProperty(deletedEntity.ContentType);
if (formModelProperty == null)
{
return;
}
var index = IndexHelper.GetIndex(deletedEntity.Id);
index.Delete();
}
catch (Exception ex)
{
Log.Error(ex, "Could not delete the index for deleted content with ID: {0}", deletedEntity.Id);
}
}
public FormData GetSubmittedValues(IPublishedContent content, int page = 1, int perPage = 10, string sortField = null, bool sortDescending = false)
{
var index = IndexHelper.GetIndex(content.Id);
var result = index.Get(sortField, sortDescending, perPage, (page - 1) * perPage) ?? Result.Empty(sortField, sortDescending);
var fields = AllValueFields();
var rows = ExtractSubmittedValues(result, fields, (field, value, row) => value == null ? null : field.FormatValueForFrontend(value, content, row.Id));
return(new FormData
{
TotalRows = result.TotalRows,
SortDescending = result.SortDescending,
SortField = result.SortField,
Rows = rows,
Fields = fields.Select(f => ToDataField(null, f, null))
});
}
public FieldValueFrequencyStatistics <IStatisticsField> GetFieldValueFrequencyStatistics(IPublishedContent content, IEnumerable <string> fieldNames = null)
{
if (content == null)
{
return(new FieldValueFrequencyStatistics <IStatisticsField>(0));
}
var fields = AllValueFields().StatisticsFields();
if (fieldNames != null)
{
fieldNames = fields.StatisticsFieldNames().Intersect(fieldNames, StringComparer.OrdinalIgnoreCase);
}
else
{
fieldNames = fields.StatisticsFieldNames();
}
if (fieldNames.Any() == false)
{
return(new FieldValueFrequencyStatistics <IStatisticsField>(0));
}
var index = IndexHelper.GetIndex(content.Id) as IStatisticsIndex;
if (index == null)
{
return(new FieldValueFrequencyStatistics <IStatisticsField>(0));
}
var statistics = index.GetFieldValueFrequencyStatistics(fieldNames);
// the statistics are indexed by field.FormSafeName - we need to reindex them by
// the fields themselves to support the frontend rendering
var result = new FieldValueFrequencyStatistics <IStatisticsField>(statistics.TotalRows);
foreach (var fieldValueFrequency in statistics.FieldValueFrequencies)
{
var field = fields.FirstOrDefault(f => f.FormSafeName == fieldValueFrequency.Field);
if (field == null)
{
continue;
}
result.Add(field, fieldValueFrequency.Frequencies);
}
return(result);
}
public HttpResponseMessage DownloadData(int id, string rowIds)
{
var document = ContentHelper.GetById(id);
var model = ContentHelper.GetFormModel(document);
if (document == null)
{
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
if (model == null)
{
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var allFields = PropertyEditorController.GetAllFieldsForDisplay(model, document);
var selectedRows = string.IsNullOrEmpty(rowIds) ? null : rowIds.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries).Select(Guid.Parse).ToList();
var index = IndexHelper.GetIndex(id);
List <Storage.Row> rows = null;
if (selectedRows != null && selectedRows.Any())
{
rows = index.Get(selectedRows).ToList();
}
else
{
var result = index.Get("_created", false, 100000, 0);
if (result != null && result.Rows != null)
{
rows = result.Rows.ToList();
}
}
if (rows == null || rows.Any() == false)
{
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var csv = new CsvExport();
foreach (var row in rows)
{
csv.AddRow();
// add date to blank header column
csv[""] = row.CreatedDate;
foreach (var modelField in allFields)
{
if (row.Fields.Any(f => f.Key == modelField.FormSafeName))
{
csv[modelField.Name] = modelField.FormatValueForCsvExport(
row.Fields.First(f => f.Key == modelField.FormSafeName).Value,
document,
row.Id
);
}
else
{
csv[modelField.Name] = string.Empty;
}
}
}
var response = new HttpResponseMessage(HttpStatusCode.OK);
var stream = new System.IO.MemoryStream(csv.ExportToBytes());
response.Content = new StreamContent(stream);
response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = string.Format(@"Form data {0}.csv", DateTime.Now.ToString("yyyy-MM-dd HH:mm"))
};
return(response);
}
public void RemoveData(int id, [FromBody] IEnumerable <Guid> ids)
{
var index = IndexHelper.GetIndex(id);
index.Remove(ids);
}