public IEnumerable <string> GetAllFieldsToExclude(string recordType) { var fields = new List <string>(); fields.AddRange(ExtractUtility.GetSystemFieldsToExclude()); fields.AddRange(ExtendedFieldsToExclude .Where(f => f.RecordType.Key == recordType) .Select(f => f.RecordField.Key)); if (!IncludeCreatedByAndOn) { fields.AddRange(new[] { "createdon", "createdby" }); } if (!IncludeModifiedByAndOn) { fields.AddRange(new[] { "modifiedon", "modifiedby" }); } if (!IncludeCrmOwner) { fields.Add("ownerid"); fields.Add("owninguser"); fields.Add("owningteam"); } if (!IncludeState) { fields.Add("statecode"); } if (!IncludeStatus) { fields.Add("statuscode"); } return(fields); }
private IEnumerable <string> GetSearchRecordTypes(TextSearchContainer container) { var typesToSearch = new List <string>(); if (!container.Request.SearchAllTypes) { typesToSearch.AddRange(container.Request.TypesToSearch.Select(r => r.RecordType.Key)); } else { typesToSearch.AddRange(Service.GetAllRecordTypes().Where(r => Service.GetRecordTypeMetadata(r).Searchable)); var systemExcludes = ExtractUtility.GetSystemRecordTypesToExclude(); if (systemExcludes != null) { typesToSearch = typesToSearch.Except(systemExcludes).ToList(); } if (container.Request.OtherExclusions != null) { typesToSearch = typesToSearch.Except(container.Request.OtherExclusions.Select(r => r.RecordType.Key)).ToList(); } if (container.Request.ExcludeEmails) { typesToSearch.RemoveAll(s => s == "email"); } if (container.Request.ExcludeEmails) { typesToSearch.RemoveAll(s => s == "post"); } } return(typesToSearch.OrderBy(n => Service.GetDisplayName(n)).ToArray()); }
private List <string> GetFieldsToExlcude(TextSearchContainer container, string recordType) { var fieldsToExclude = new List <string>(); fieldsToExclude.AddRange(ExtractUtility.GetSystemFieldsToExclude()); if (container.Request.FieldExclusions != null) { fieldsToExclude.AddRange(container.Request.FieldExclusions.Where(fe => fe.RecordType.Key == recordType).Select(fe => fe.RecordField.Key)); } foreach (var field in Service.GetFields(recordType)) { var fieldType = Service.GetFieldType(field, recordType); if (fieldType == RecordFieldType.Uniqueidentifier) { fieldsToExclude.Add(field); } else if (Service.GetFieldType(field, recordType) == RecordFieldType.String && Service.GetFieldMetadata(field, recordType).TextFormat == TextFormat.PhoneticGuide) { fieldsToExclude.Add(field); } if (field.EndsWith("_base") && fieldType == RecordFieldType.Money) { fieldsToExclude.Add(field); } } return(fieldsToExclude); }
static void ExtractDLLs(ODBFile odb, string fileName) { List <byte[]> extractedDlls = ExtractUtility.ExtractDLLs(odb.ODBBinary); for (int i = 0; i < extractedDlls.Count; i++) { string preferredName = $"{Path.GetFileNameWithoutExtension(fileName)}_{i}.dll"; Console.WriteLine(preferredName); File.WriteAllBytes(preferredName, extractedDlls[i]); } Console.WriteLine($"{extractedDlls.Count} DLLs extracted"); }
static void ExtractZips(ODBFile odb, string fileName) { List <byte[]> extractedZips = ExtractUtility.ExtractZips(odb.ODBBinary); for (int i = 0; i < extractedZips.Count; i++) { string preferredName = ExtractUtility.GenerateNameForZipFile(extractedZips[i], i); Console.WriteLine(preferredName); File.WriteAllBytes($"{Path.GetFileNameWithoutExtension(fileName)}_{preferredName}", extractedZips[i]); } Console.WriteLine($"{extractedZips.Count} ZIP/JARs extracted"); }
private IRecord ExtractMainRecord(RecordExtractContainer container) { container.Controller.UpdateProgress(1, 3, "Loading Record"); var recordToExtract = Service.Get(container.Request.RecordLookup.RecordType, container.Request.RecordLookup.Id); container.RecordToExtract = recordToExtract; var primaryField = Service.GetPrimaryField(recordToExtract.Type); var recordName = recordToExtract.GetStringField(primaryField); container.AddBookmark(ExtractUtility.CheckStripFormatting(recordName, container.RecordToExtractType, primaryField)); container.Response.Record = recordToExtract; container.Controller.UpdateProgress(2, 3, "Loading Record"); WriteRecordToSection(recordToExtract, container.Section, DetailLevel.AllFields, container.Request); return(recordToExtract); }
public IEnumerable <string> GetRecordTypesToExclude() { return (ExtractUtility.GetSystemRecordTypesToExclude() .Union(ExtendedRecordTypesToExclude.Select(r => r.RecordType.Key))); }
private void AppendStringFieldMatches(TextSearchContainer container, string recordType, int done, int count, Dictionary <string, IRecord> recordsToOutput) { var thisRecordType = recordType; container.Controller.UpdateProgress(done, count, string.Format("Searching String Fields In {0}", Service.GetCollectionName(recordType))); try { var fieldsToExclude = GetFieldsToExlcude(container, recordType); var stringFields = Service.GetFields(recordType) .Where(f => Service.IsString(f, recordType)) .Where(f => Service.IsString(f, thisRecordType)) .Except(fieldsToExclude) .ToArray(); if (stringFields.Any()) { var htmlSearchFields = container.Request.StripHtmlTagsPriorToSearch ? stringFields .Where(s => IsHtmlField(recordType, s, container)).ToArray() : new string[0]; var setSearchFields = stringFields .Intersect(ExtractUtility.GetSystemTextSearchSetFields() .Where(f => f.RecordType.Key == recordType) .Select(f => f.RecordField.Key) .Except(htmlSearchFields)) .ToArray(); var nonSetSearchFields = stringFields .Except(setSearchFields) .Except(htmlSearchFields) .ToArray(); if (htmlSearchFields.Any()) { //this code written as the crm web service / sql timedout when doing text searches over the entire record table //i thus split all the records into sets defined by a date range and query the text in each set iteratively //this way I limit the volume of text being searched in each crm web service query by a approximate number of records defined in the settings int totalCount = 0; var sortedDatesTemplate = GetDateRangesForSetSearches(recordType, container, out totalCount); var totalDone = 0; foreach (var field in htmlSearchFields) { try { var thisFieldSortedDates = sortedDatesTemplate.ToList(); var label = Service.GetFieldLabel(field, recordType); //now query the text in each date range set while (thisFieldSortedDates.Any()) { var first = thisFieldSortedDates.First(); var limit = thisFieldSortedDates.Count > 1 ? thisFieldSortedDates[1] : (DateTime?)null; if (first.Equals(limit) && thisFieldSortedDates.Any(l => l > first)) { limit = thisFieldSortedDates.First(l => l > first); } var query = new QueryDefinition(recordType); query.RootFilter = new Filter(); query.RootFilter.ConditionOperator = container.Request.Operator == TextSearchRequest.SearchTermOperator.And ? FilterOperator.And : FilterOperator.Or; query.RootFilter.SubFilters = container.Request.SearchTerms.Select(s => { var searchTermFilter = new Filter(); searchTermFilter.Conditions = new List <Condition>(); searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.GreaterEqual, first)); if (limit.HasValue) { searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.LessThan, limit.Value)); } return(searchTermFilter); }).ToList(); var allOfType = Service.RetreiveAll(query); foreach (var item in allOfType) { container.Controller.UpdateLevel2Progress(totalDone++, totalCount, string.Format("Searching Html {0}", label)); var fieldValue = item.GetStringField(field); if (fieldValue != null) { var stripHtml = fieldValue.StripHtml(); if (IsSearchMatch(stripHtml, container)) { recordsToOutput.Add(item.Id, item); container.AddMatchedRecord(field, item); } } } thisFieldSortedDates.RemoveAt(0); } } catch (Exception ex) { container.Response.AddResponseItem( new TextSearchResponseItem("Error Searching Html Fields", recordType, field, ex)); } } } var fieldsTodo = nonSetSearchFields.Count(); var fieldsDone = 0; foreach (var field in nonSetSearchFields.OrderBy(f => Service.GetFieldLabel(f, recordType))) { container.Controller.UpdateLevel2Progress(fieldsDone++, fieldsTodo, $"Searching {Service.GetFieldLabel(field, recordType)}"); try { var conditions = container.Request.SearchTerms.Select(s => new Condition(field, ConditionType.Like, string.Format("%{0}%", s.Text))) .ToArray(); var stringFieldMatches = (container.Request.Operator == TextSearchRequest.SearchTermOperator.And ? Service.RetrieveAllAndClauses(recordType, conditions, null) : Service.RetrieveAllOrClauses(recordType, conditions, null)) .ToArray(); foreach (var stringFieldMatch in stringFieldMatches) { container.AddMatchedRecord(field, stringFieldMatch); if (!recordsToOutput.ContainsKey(stringFieldMatch.Id)) { recordsToOutput.Add(stringFieldMatch.Id, stringFieldMatch); } } } catch (Exception ex) { container.Response.AddResponseItem( new TextSearchResponseItem("Error Searching String Fields", recordType, field, ex)); } } if (setSearchFields.Any()) { //this code written as the crm web service / sql timedout when doing text searches over the entire record table //i thus split all the records into sets defined by a date range and query the text in each set iteratively //this way I limit the volume of text being searched in each crm web service query by a approximate number of records defined in the settings container.Controller.UpdateLevel2Progress(0, 1, string.Format("Configuring Search Sets")); var fieldSetsTodo = 0; var sortedDatesTemplate = GetDateRangesForSetSearches(recordType, container, out fieldSetsTodo); foreach (var field in setSearchFields) { try { var thisFieldSortedDates = sortedDatesTemplate.ToList(); var fieldSetsDone = 0; var label = Service.GetFieldLabel(field, recordType); //now query the text in each date range set while (thisFieldSortedDates.Any()) { container.Controller.UpdateLevel2Progress(fieldSetsDone + ExtractUtility.TextSearchSetSize, fieldSetsTodo, string.Format("Searching {0}", label)); var first = thisFieldSortedDates.First(); var limit = thisFieldSortedDates.Count > 1 ? thisFieldSortedDates[1] : (DateTime?)null; var query = new QueryDefinition(recordType); query.RootFilter = new Filter(); query.RootFilter.ConditionOperator = container.Request.Operator == TextSearchRequest.SearchTermOperator.And ? FilterOperator.And : FilterOperator.Or; query.RootFilter.SubFilters = container.Request.SearchTerms.Select(s => { var searchTermFilter = new Filter(); searchTermFilter.Conditions = new List <Condition>(); searchTermFilter.Conditions.Add(new Condition(field, ConditionType.Like, string.Format("%{0}%", s.Text))); searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.GreaterEqual, first)); if (limit.HasValue) { searchTermFilter.Conditions.Add(new Condition("createdon", ConditionType.LessThan, limit.Value)); } return(searchTermFilter); }).ToList(); var stringFieldMatches = Service.RetreiveAll(query); foreach (var stringFieldMatch in stringFieldMatches) { container.AddMatchedRecord(field, stringFieldMatch); if (!recordsToOutput.ContainsKey(stringFieldMatch.Id)) { recordsToOutput.Add(stringFieldMatch.Id, stringFieldMatch); } } thisFieldSortedDates.RemoveAt(0); } } catch (Exception ex) { container.Response.AddResponseItem( new TextSearchResponseItem("Error Searching String Fields", recordType, field, ex)); } } } } } catch (Exception ex) { container.Response.AddResponseItem( new TextSearchResponseItem("Error Searching String Fields", recordType, ex)); } }
private bool IsHtmlField(string recordType, string fieldName, TextSearchContainer container) { var fields = ExtractUtility.GetSystemHtmlFields().Union(container.Request.CustomHtmlFields ?? new RecordFieldSetting[0]); return(fields.Any(f => f.RecordType.Key == recordType && f.RecordField.Key == fieldName)); }
private void AppendFieldMatchesToDocument(TextSearchContainer container, Dictionary <string, IRecord> recordsToOutput, string recordType, ContentBookmark bookmark) { Table2Column fieldCountTable = null; var fieldsDictionary = new Dictionary <string, int>(); if (recordsToOutput.Any()) { var recordOutput = false; var todoDone = 0; var todoCount = recordsToOutput.Count; var primaryField = Service.GetPrimaryField(recordType); var fieldsToExclude = GetFieldsToExlcude(container, recordType); //some lookup names dont get loaded into the record so will load them all now so i don't have to field by field recordsToOutput.Values.PopulateEmptyLookups(Service, ExtractUtility.GetSystemRecordTypesToExclude()); var primaryFieldLabel = Service.GetFieldLabel(primaryField, recordType); var recordTypeCollectionLabel = Service.GetCollectionName(recordType); foreach (var match in recordsToOutput.Values) { container.Controller.UpdateLevel2Progress(todoDone++, todoCount, string.Format("Appending {0} To Document", recordTypeCollectionLabel)); var fieldsToDisplay = new List <string>(); foreach (var field in match.GetFieldsInEntity().Where(f => !fieldsToExclude.Contains(f))) { if (Service.IsString(field, recordType)) { var value = Service.GetFieldAsDisplayString(match, field); if (container.Request.StripHtmlTagsPriorToSearch && IsHtmlField(recordType, field, container)) { value = value.StripHtml(); } if (value != null) { if (IsSearchMatch(value, container)) { fieldsToDisplay.Add(field); if (!fieldsDictionary.ContainsKey(field)) { fieldsDictionary.Add(field, 0); } fieldsDictionary[field]++; } } } } if (fieldsToDisplay.Any()) { if (!recordOutput) { var thisBookmark = container.Section.AddHeading2WithBookmark(string.Format("{0} ({1})", Service.GetCollectionName(recordType), recordsToOutput.Count())); bookmark.AddChildBookmark(thisBookmark); recordOutput = true; fieldCountTable = container.Section.Add2ColumnTable(); } var table = container.Section.Add2ColumnTable(); table.AddFieldToTable(primaryFieldLabel, match.GetStringField(primaryField)); foreach (var field in fieldsToDisplay) { var value = Service.GetFieldAsDisplayString(match, field); if (container.Request.StripHtmlTagsPriorToSearch && IsHtmlField(recordType, field, container)) { value = value.StripHtml(); } if (value != null) { if (IsSearchMatch(value, container)) { table.AddFieldToTable(Service.GetFieldLabel(field, recordType), value); } } } } } //insert field count summary if (fieldsDictionary.Any()) { foreach (var field in fieldsDictionary .OrderBy(f => Service.GetFieldLabel(f.Key, recordType))) { fieldCountTable.AddFieldToTable(Service.GetFieldLabel(field.Key, recordType), field.Value.ToString()); } } } }
public IEnumerable <string> GetStringValuesToExclude() { return(ExtractUtility.GetStringValuesToExclude()); }
private bool IsHtmlField(string recordType, string fieldName, RecordExtractToDocumentRequest request) { var fields = ExtractUtility.GetSystemHtmlFields().Union(request.CustomHtmlFields ?? new RecordFieldSetting[0]); return(fields.Any(f => f.RecordType.Key == recordType && f.RecordField.Key == fieldName)); }
private void WriteRecordToSection(IRecord record, Section section, DetailLevel detailLevel, RecordExtractToDocumentRequest request) { if (detailLevel == DetailLevel.CountsOnly) { return; } var primaryField = Service.GetPrimaryField(record.Type); var recordName = Service.GetFieldAsDisplayString(record, primaryField); var table = section.Add2ColumnTable(); table.AddFieldToTable(Service.GetFieldLabel(primaryField, record.Type), ExtractUtility.CheckStripFormatting(recordName, record.Type, primaryField)); if (detailLevel == DetailLevel.Names || request.OnlyDisplayName(record.Type)) { return; } var fields = Service.GetFields(record.Type).Intersect(record.GetFieldsInEntity()).ToArray(); if (fields.Any()) { var primaryKey = Service.GetPrimaryKey(record.Type); var actualFieldsToExclude = new List <string> { primaryField, primaryKey }; actualFieldsToExclude.AddRange(request.GetAllFieldsToExclude(record.Type)); foreach (var field in fields) { var fieldType = Service.GetFieldType(field, record.Type); if (fieldType == RecordFieldType.Uniqueidentifier) { actualFieldsToExclude.Add(field); } else if (Service.GetFieldType(field, record.Type) == RecordFieldType.String && new[] { TextFormat.PhoneticGuide, TextFormat.VersionNumber }.Contains( Service.GetFieldMetadata(field, record.Type).TextFormat)) { actualFieldsToExclude.Add(field); } if (field.EndsWith("_base") && fields.Contains(field.Left(field.Length - 5))) { actualFieldsToExclude.Add(field); } } var orderedFieldsToDisplay = fields .Where(f => !actualFieldsToExclude.Contains(f)) .OrderBy(f => Service.GetFieldLabel(f, record.Type)); foreach (var field in orderedFieldsToDisplay) { var label = Service.GetFieldLabel(field, record.Type); var display = Service.GetFieldAsDisplayString(record, field); if (request.StripHtmlTags && IsHtmlField(record.Type, field, request)) { display = display.StripHtml(); } if (!label.IsNullOrWhiteSpace() && !display.IsNullOrWhiteSpace() && !GetStringValuesToExclude().Contains(display)) { table.AddFieldToTable(label, display); } } } }
private IEnumerable <string> GetRelationshipsToExclude() { return(ExtractUtility.GetSystemRelationshipsToExclude()); }
private IEnumerable <string> GetRecordTypesToExclude() { return(ExtractUtility.GetSystemRecordTypesToExclude()); }
public IEnumerable <string> GetFieldsToExclude() { return (ExtractUtility.GetSystemFieldsToExclude().Union(ExtendedFieldsToExclude.Select(r => r.RecordField.Key))); }
public IEnumerable <RecordFieldSetting> GetTextSearchSetFields() { return(ExtractUtility.GetSystemTextSearchSetFields().Union(ExtendedTextSearchSetFields)); }
void ProcessRecordsContainedInName(TextSearchContainer container) { var bookmark = container.AddHeadingWithBookmark("Records With Matching Name"); var recordTypes = GetSearchRecordTypes().ToArray(); var count = recordTypes.Count(); var done = 0; foreach (var recordType in recordTypes) { try { var progressTextPrefix = string.Format("Searching Record Names In {0}", Service.GetCollectionName(recordType)); container.Controller.UpdateProgress(done++, count, progressTextPrefix); var primaryField = Service.GetPrimaryField(recordType); if (!primaryField.IsNullOrWhiteSpace()) { var conditions = new[] { new Condition(primaryField, ConditionType.Like, string.Format("%{0}%", container.Request.SearchText)) }; var matches = Service.RetrieveAllAndClauses(recordType, conditions, new[] { primaryField }).ToArray(); if (matches.Any()) { try { var thisBookmark = container.Section.AddHeading2WithBookmark(string.Format("{0} ({1})", Service.GetCollectionName(recordType), matches.Count())); bookmark.AddChildBookmark(thisBookmark); var table = container.Section.Add1ColumnTable(); var matchCount = matches.Count(); var matchCountDone = 0; foreach (var match in matches) { container.Controller.UpdateProgress(done, count, string.Format("{0} (Adding {1} Of {2})", progressTextPrefix, ++matchCountDone, matchCount)); container.AddNameMatch(match); var outputText = match.GetStringField(primaryField); outputText = ExtractUtility.CheckStripFormatting(outputText, recordType, primaryField); table.AddRow(outputText); } } catch (Exception ex) { container.Response.AddResponseItem( new TextSearchResponseItem("Error Adding Matched Record", recordType, ex)); } } } } catch (Exception ex) { container.Response.AddResponseItem(new TextSearchResponseItem("Error Adding Match Records", recordType, ex)); } } }