public IEnumerable <string> GetAllFieldsToExclude(string recordType)
{
var fields = new List <string>();
fields.AddRange(ExtractUtility.GetSystemFieldsToExclude());
fields.AddRange(ExtendedFieldsToExclude
.Where(f => f.RecordType.Key == recordType)
.Select(f => f.RecordField.Key));
if (!IncludeCreatedByAndOn)
{
fields.AddRange(new[] { "createdon", "createdby" });
}
if (!IncludeModifiedByAndOn)
{
fields.AddRange(new[] { "modifiedon", "modifiedby" });
}
if (!IncludeCrmOwner)
{
fields.Add("ownerid");
fields.Add("owninguser");
fields.Add("owningteam");
}
if (!IncludeState)
{
fields.Add("statecode");
}
if (!IncludeStatus)
{
fields.Add("statuscode");
}
return(fields);
}
private IEnumerable <string> GetSearchRecordTypes(TextSearchContainer container)
{
var typesToSearch = new List <string>();
if (!container.Request.SearchAllTypes)
{
typesToSearch.AddRange(container.Request.TypesToSearch.Select(r => r.RecordType.Key));
}
else
{
typesToSearch.AddRange(Service.GetAllRecordTypes().Where(r => Service.GetRecordTypeMetadata(r).Searchable));
var systemExcludes = ExtractUtility.GetSystemRecordTypesToExclude();
if (systemExcludes != null)
{
typesToSearch = typesToSearch.Except(systemExcludes).ToList();
}
if (container.Request.OtherExclusions != null)
{
typesToSearch = typesToSearch.Except(container.Request.OtherExclusions.Select(r => r.RecordType.Key)).ToList();
}
if (container.Request.ExcludeEmails)
{
typesToSearch.RemoveAll(s => s == "email");
}
if (container.Request.ExcludeEmails)
{
typesToSearch.RemoveAll(s => s == "post");
}
}
return(typesToSearch.OrderBy(n => Service.GetDisplayName(n)).ToArray());
}
private List <string> GetFieldsToExlcude(TextSearchContainer container, string recordType)
{
var fieldsToExclude = new List <string>();
fieldsToExclude.AddRange(ExtractUtility.GetSystemFieldsToExclude());
if (container.Request.FieldExclusions != null)
{
fieldsToExclude.AddRange(container.Request.FieldExclusions.Where(fe => fe.RecordType.Key == recordType).Select(fe => fe.RecordField.Key));
}
foreach (var field in Service.GetFields(recordType))
{
var fieldType = Service.GetFieldType(field, recordType);
if (fieldType == RecordFieldType.Uniqueidentifier)
{
fieldsToExclude.Add(field);
}
else if (Service.GetFieldType(field, recordType) == RecordFieldType.String &&
Service.GetFieldMetadata(field, recordType).TextFormat == TextFormat.PhoneticGuide)
{
fieldsToExclude.Add(field);
}
if (field.EndsWith("_base") && fieldType == RecordFieldType.Money)
{
fieldsToExclude.Add(field);
}
}
return(fieldsToExclude);
}
static void ExtractDLLs(ODBFile odb, string fileName)
{
List <byte[]> extractedDlls = ExtractUtility.ExtractDLLs(odb.ODBBinary);
for (int i = 0; i < extractedDlls.Count; i++)
{
string preferredName = $"{Path.GetFileNameWithoutExtension(fileName)}_{i}.dll";
Console.WriteLine(preferredName);
File.WriteAllBytes(preferredName, extractedDlls[i]);
}
Console.WriteLine($"{extractedDlls.Count} DLLs extracted");
}
static void ExtractZips(ODBFile odb, string fileName)
{
List <byte[]> extractedZips = ExtractUtility.ExtractZips(odb.ODBBinary);
for (int i = 0; i < extractedZips.Count; i++)
{
string preferredName = ExtractUtility.GenerateNameForZipFile(extractedZips[i], i);
Console.WriteLine(preferredName);
File.WriteAllBytes($"{Path.GetFileNameWithoutExtension(fileName)}_{preferredName}", extractedZips[i]);
}
Console.WriteLine($"{extractedZips.Count} ZIP/JARs extracted");
}
private IRecord ExtractMainRecord(RecordExtractContainer container)
{
container.Controller.UpdateProgress(1, 3, "Loading Record");
var recordToExtract = Service.Get(container.Request.RecordLookup.RecordType,
container.Request.RecordLookup.Id);
container.RecordToExtract = recordToExtract;
var primaryField = Service.GetPrimaryField(recordToExtract.Type);
var recordName = recordToExtract.GetStringField(primaryField);
container.AddBookmark(ExtractUtility.CheckStripFormatting(recordName, container.RecordToExtractType,
primaryField));
container.Response.Record = recordToExtract;
container.Controller.UpdateProgress(2, 3, "Loading Record");
WriteRecordToSection(recordToExtract, container.Section, DetailLevel.AllFields, container.Request);
return(recordToExtract);
}
public IEnumerable <string> GetRecordTypesToExclude()
{
return
(ExtractUtility.GetSystemRecordTypesToExclude()
.Union(ExtendedRecordTypesToExclude.Select(r => r.RecordType.Key)));
}
private void AppendStringFieldMatches(TextSearchContainer container, string recordType, int done, int count,
Dictionary <string, IRecord> recordsToOutput)
{
var thisRecordType = recordType;
container.Controller.UpdateProgress(done, count,
string.Format("Searching String Fields In {0}", Service.GetCollectionName(recordType)));
try
{
var fieldsToExclude = GetFieldsToExlcude(container, recordType);
var stringFields = Service.GetFields(recordType)
.Where(f => Service.IsString(f, recordType))
.Where(f => Service.IsString(f, thisRecordType))
.Except(fieldsToExclude)
.ToArray();
if (stringFields.Any())
{
var htmlSearchFields = container.Request.StripHtmlTagsPriorToSearch
? stringFields
.Where(s => IsHtmlField(recordType, s, container)).ToArray()
: new string[0];
var setSearchFields = stringFields
.Intersect(ExtractUtility.GetSystemTextSearchSetFields()
.Where(f => f.RecordType.Key == recordType)
.Select(f => f.RecordField.Key)
.Except(htmlSearchFields))
.ToArray();
var nonSetSearchFields = stringFields
.Except(setSearchFields)
.Except(htmlSearchFields)
.ToArray();
if (htmlSearchFields.Any())
{
//this code written as the crm web service / sql timedout when doing text searches over the entire record table
//i thus split all the records into sets defined by a date range and query the text in each set iteratively
//this way I limit the volume of text being searched in each crm web service query by a approximate number of records defined in the settings
int totalCount = 0;
var sortedDatesTemplate = GetDateRangesForSetSearches(recordType, container, out totalCount);
var totalDone = 0;
foreach (var field in htmlSearchFields)
{
try
{
var thisFieldSortedDates = sortedDatesTemplate.ToList();
var label = Service.GetFieldLabel(field, recordType);
//now query the text in each date range set
while (thisFieldSortedDates.Any())
{
var first = thisFieldSortedDates.First();
var limit = thisFieldSortedDates.Count > 1 ? thisFieldSortedDates[1] : (DateTime?)null;
if (first.Equals(limit) && thisFieldSortedDates.Any(l => l > first))
{
limit = thisFieldSortedDates.First(l => l > first);
}
var query = new QueryDefinition(recordType);
query.RootFilter = new Filter();
query.RootFilter.ConditionOperator = container.Request.Operator == TextSearchRequest.SearchTermOperator.And ? FilterOperator.And : FilterOperator.Or;
query.RootFilter.SubFilters = container.Request.SearchTerms.Select(s =>
{
var searchTermFilter = new Filter();
searchTermFilter.Conditions = new List <Condition>();
searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.GreaterEqual, first));
if (limit.HasValue)
{
searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.LessThan, limit.Value));
}
return(searchTermFilter);
}).ToList();
var allOfType = Service.RetreiveAll(query);
foreach (var item in allOfType)
{
container.Controller.UpdateLevel2Progress(totalDone++, totalCount, string.Format("Searching Html {0}", label));
var fieldValue = item.GetStringField(field);
if (fieldValue != null)
{
var stripHtml = fieldValue.StripHtml();
if (IsSearchMatch(stripHtml, container))
{
recordsToOutput.Add(item.Id, item);
container.AddMatchedRecord(field, item);
}
}
}
thisFieldSortedDates.RemoveAt(0);
}
}
catch (Exception ex)
{
container.Response.AddResponseItem(
new TextSearchResponseItem("Error Searching Html Fields", recordType, field, ex));
}
}
}
var fieldsTodo = nonSetSearchFields.Count();
var fieldsDone = 0;
foreach (var field in nonSetSearchFields.OrderBy(f => Service.GetFieldLabel(f, recordType)))
{
container.Controller.UpdateLevel2Progress(fieldsDone++, fieldsTodo, $"Searching {Service.GetFieldLabel(field, recordType)}");
try
{
var conditions =
container.Request.SearchTerms.Select(s =>
new Condition(field, ConditionType.Like, string.Format("%{0}%", s.Text)))
.ToArray();
var stringFieldMatches = (container.Request.Operator == TextSearchRequest.SearchTermOperator.And
? Service.RetrieveAllAndClauses(recordType, conditions, null)
: Service.RetrieveAllOrClauses(recordType, conditions, null))
.ToArray();
foreach (var stringFieldMatch in stringFieldMatches)
{
container.AddMatchedRecord(field, stringFieldMatch);
if (!recordsToOutput.ContainsKey(stringFieldMatch.Id))
{
recordsToOutput.Add(stringFieldMatch.Id, stringFieldMatch);
}
}
}
catch (Exception ex)
{
container.Response.AddResponseItem(
new TextSearchResponseItem("Error Searching String Fields", recordType, field, ex));
}
}
if (setSearchFields.Any())
{
//this code written as the crm web service / sql timedout when doing text searches over the entire record table
//i thus split all the records into sets defined by a date range and query the text in each set iteratively
//this way I limit the volume of text being searched in each crm web service query by a approximate number of records defined in the settings
container.Controller.UpdateLevel2Progress(0, 1, string.Format("Configuring Search Sets"));
var fieldSetsTodo = 0;
var sortedDatesTemplate = GetDateRangesForSetSearches(recordType, container, out fieldSetsTodo);
foreach (var field in setSearchFields)
{
try
{
var thisFieldSortedDates = sortedDatesTemplate.ToList();
var fieldSetsDone = 0;
var label = Service.GetFieldLabel(field, recordType);
//now query the text in each date range set
while (thisFieldSortedDates.Any())
{
container.Controller.UpdateLevel2Progress(fieldSetsDone + ExtractUtility.TextSearchSetSize, fieldSetsTodo,
string.Format("Searching {0}", label));
var first = thisFieldSortedDates.First();
var limit = thisFieldSortedDates.Count > 1 ? thisFieldSortedDates[1] : (DateTime?)null;
var query = new QueryDefinition(recordType);
query.RootFilter = new Filter();
query.RootFilter.ConditionOperator = container.Request.Operator == TextSearchRequest.SearchTermOperator.And ? FilterOperator.And : FilterOperator.Or;
query.RootFilter.SubFilters = container.Request.SearchTerms.Select(s =>
{
var searchTermFilter = new Filter();
searchTermFilter.Conditions = new List <Condition>();
searchTermFilter.Conditions.Add(new Condition(field, ConditionType.Like, string.Format("%{0}%", s.Text)));
searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.GreaterEqual, first));
if (limit.HasValue)
{
searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.LessThan, limit.Value));
}
return(searchTermFilter);
}).ToList();
var stringFieldMatches = Service.RetreiveAll(query);
foreach (var stringFieldMatch in stringFieldMatches)
{
container.AddMatchedRecord(field, stringFieldMatch);
if (!recordsToOutput.ContainsKey(stringFieldMatch.Id))
{
recordsToOutput.Add(stringFieldMatch.Id, stringFieldMatch);
}
}
thisFieldSortedDates.RemoveAt(0);
}
}
catch (Exception ex)
{
container.Response.AddResponseItem(
new TextSearchResponseItem("Error Searching String Fields", recordType, field, ex));
}
}
}
}
}
catch (Exception ex)
{
container.Response.AddResponseItem(
new TextSearchResponseItem("Error Searching String Fields",
recordType, ex));
}
}
private bool IsHtmlField(string recordType, string fieldName, TextSearchContainer container)
{
var fields = ExtractUtility.GetSystemHtmlFields().Union(container.Request.CustomHtmlFields ?? new RecordFieldSetting[0]);
return(fields.Any(f => f.RecordType.Key == recordType && f.RecordField.Key == fieldName));
}
private void AppendFieldMatchesToDocument(TextSearchContainer container,
Dictionary <string, IRecord> recordsToOutput, string recordType,
ContentBookmark bookmark)
{
Table2Column fieldCountTable = null;
var fieldsDictionary = new Dictionary <string, int>();
if (recordsToOutput.Any())
{
var recordOutput = false;
var todoDone = 0;
var todoCount = recordsToOutput.Count;
var primaryField = Service.GetPrimaryField(recordType);
var fieldsToExclude = GetFieldsToExlcude(container, recordType);
//some lookup names dont get loaded into the record so will load them all now so i don't have to field by field
recordsToOutput.Values.PopulateEmptyLookups(Service, ExtractUtility.GetSystemRecordTypesToExclude());
var primaryFieldLabel = Service.GetFieldLabel(primaryField, recordType);
var recordTypeCollectionLabel = Service.GetCollectionName(recordType);
foreach (var match in recordsToOutput.Values)
{
container.Controller.UpdateLevel2Progress(todoDone++, todoCount,
string.Format("Appending {0} To Document", recordTypeCollectionLabel));
var fieldsToDisplay = new List <string>();
foreach (var field in match.GetFieldsInEntity().Where(f => !fieldsToExclude.Contains(f)))
{
if (Service.IsString(field, recordType))
{
var value = Service.GetFieldAsDisplayString(match, field);
if (container.Request.StripHtmlTagsPriorToSearch &&
IsHtmlField(recordType, field, container))
{
value = value.StripHtml();
}
if (value != null)
{
if (IsSearchMatch(value, container))
{
fieldsToDisplay.Add(field);
if (!fieldsDictionary.ContainsKey(field))
{
fieldsDictionary.Add(field, 0);
}
fieldsDictionary[field]++;
}
}
}
}
if (fieldsToDisplay.Any())
{
if (!recordOutput)
{
var thisBookmark =
container.Section.AddHeading2WithBookmark(string.Format("{0} ({1})", Service.GetCollectionName(recordType), recordsToOutput.Count()));
bookmark.AddChildBookmark(thisBookmark);
recordOutput = true;
fieldCountTable = container.Section.Add2ColumnTable();
}
var table = container.Section.Add2ColumnTable();
table.AddFieldToTable(primaryFieldLabel, match.GetStringField(primaryField));
foreach (var field in fieldsToDisplay)
{
var value = Service.GetFieldAsDisplayString(match, field);
if (container.Request.StripHtmlTagsPriorToSearch &&
IsHtmlField(recordType, field, container))
{
value = value.StripHtml();
}
if (value != null)
{
if (IsSearchMatch(value, container))
{
table.AddFieldToTable(Service.GetFieldLabel(field, recordType),
value);
}
}
}
}
}
//insert field count summary
if (fieldsDictionary.Any())
{
foreach (var field in fieldsDictionary
.OrderBy(f => Service.GetFieldLabel(f.Key, recordType)))
{
fieldCountTable.AddFieldToTable(Service.GetFieldLabel(field.Key, recordType), field.Value.ToString());
}
}
}
}
public IEnumerable <string> GetStringValuesToExclude()
{
return(ExtractUtility.GetStringValuesToExclude());
}
private bool IsHtmlField(string recordType, string fieldName, RecordExtractToDocumentRequest request)
{
var fields = ExtractUtility.GetSystemHtmlFields().Union(request.CustomHtmlFields ?? new RecordFieldSetting[0]);
return(fields.Any(f => f.RecordType.Key == recordType && f.RecordField.Key == fieldName));
}
private void WriteRecordToSection(IRecord record, Section section, DetailLevel detailLevel, RecordExtractToDocumentRequest request)
{
if (detailLevel == DetailLevel.CountsOnly)
{
return;
}
var primaryField = Service.GetPrimaryField(record.Type);
var recordName = Service.GetFieldAsDisplayString(record, primaryField);
var table = section.Add2ColumnTable();
table.AddFieldToTable(Service.GetFieldLabel(primaryField, record.Type),
ExtractUtility.CheckStripFormatting(recordName, record.Type, primaryField));
if (detailLevel == DetailLevel.Names || request.OnlyDisplayName(record.Type))
{
return;
}
var fields = Service.GetFields(record.Type).Intersect(record.GetFieldsInEntity()).ToArray();
if (fields.Any())
{
var primaryKey = Service.GetPrimaryKey(record.Type);
var actualFieldsToExclude = new List <string> {
primaryField, primaryKey
};
actualFieldsToExclude.AddRange(request.GetAllFieldsToExclude(record.Type));
foreach (var field in fields)
{
var fieldType = Service.GetFieldType(field, record.Type);
if (fieldType == RecordFieldType.Uniqueidentifier)
{
actualFieldsToExclude.Add(field);
}
else if (Service.GetFieldType(field, record.Type) == RecordFieldType.String &&
new[] { TextFormat.PhoneticGuide, TextFormat.VersionNumber }.Contains(
Service.GetFieldMetadata(field, record.Type).TextFormat))
{
actualFieldsToExclude.Add(field);
}
if (field.EndsWith("_base") && fields.Contains(field.Left(field.Length - 5)))
{
actualFieldsToExclude.Add(field);
}
}
var orderedFieldsToDisplay = fields
.Where(f => !actualFieldsToExclude.Contains(f))
.OrderBy(f => Service.GetFieldLabel(f, record.Type));
foreach (var field in orderedFieldsToDisplay)
{
var label = Service.GetFieldLabel(field, record.Type);
var display = Service.GetFieldAsDisplayString(record, field);
if (request.StripHtmlTags && IsHtmlField(record.Type, field, request))
{
display = display.StripHtml();
}
if (!label.IsNullOrWhiteSpace() && !display.IsNullOrWhiteSpace() &&
!GetStringValuesToExclude().Contains(display))
{
table.AddFieldToTable(label, display);
}
}
}
}
private IEnumerable <string> GetRelationshipsToExclude()
{
return(ExtractUtility.GetSystemRelationshipsToExclude());
}
private IEnumerable <string> GetRecordTypesToExclude()
{
return(ExtractUtility.GetSystemRecordTypesToExclude());
}
public IEnumerable <string> GetFieldsToExclude()
{
return
(ExtractUtility.GetSystemFieldsToExclude().Union(ExtendedFieldsToExclude.Select(r => r.RecordField.Key)));
}
public IEnumerable <RecordFieldSetting> GetTextSearchSetFields()
{
return(ExtractUtility.GetSystemTextSearchSetFields().Union(ExtendedTextSearchSetFields));
}
void ProcessRecordsContainedInName(TextSearchContainer container)
{
var bookmark = container.AddHeadingWithBookmark("Records With Matching Name");
var recordTypes = GetSearchRecordTypes().ToArray();
var count = recordTypes.Count();
var done = 0;
foreach (var recordType in recordTypes)
{
try
{
var progressTextPrefix = string.Format("Searching Record Names In {0}",
Service.GetCollectionName(recordType));
container.Controller.UpdateProgress(done++, count, progressTextPrefix);
var primaryField = Service.GetPrimaryField(recordType);
if (!primaryField.IsNullOrWhiteSpace())
{
var conditions = new[]
{
new Condition(primaryField, ConditionType.Like,
string.Format("%{0}%", container.Request.SearchText))
};
var matches =
Service.RetrieveAllAndClauses(recordType, conditions, new[] { primaryField }).ToArray();
if (matches.Any())
{
try
{
var thisBookmark =
container.Section.AddHeading2WithBookmark(string.Format("{0} ({1})", Service.GetCollectionName(recordType), matches.Count()));
bookmark.AddChildBookmark(thisBookmark);
var table = container.Section.Add1ColumnTable();
var matchCount = matches.Count();
var matchCountDone = 0;
foreach (var match in matches)
{
container.Controller.UpdateProgress(done, count,
string.Format("{0} (Adding {1} Of {2})", progressTextPrefix, ++matchCountDone,
matchCount));
container.AddNameMatch(match);
var outputText = match.GetStringField(primaryField);
outputText = ExtractUtility.CheckStripFormatting(outputText, recordType,
primaryField);
table.AddRow(outputText);
}
}
catch (Exception ex)
{
container.Response.AddResponseItem(
new TextSearchResponseItem("Error Adding Matched Record",
recordType, ex));
}
}
}
}
catch (Exception ex)
{
container.Response.AddResponseItem(new TextSearchResponseItem("Error Adding Match Records",
recordType, ex));
}
}
}
