public void WriteCustomFieldConfigForOneFieldSourceType(
int classId,
string fieldSourceType, // Can be either "entry", "senses", or "examples"
Dictionary <string, LfConfigFieldBase> lfCustomFieldConfig,
Dictionary <string, string> lfCustomFieldTypes
)
{
IEnumerable <int> customFieldIds =
LcmMetaData.GetFields(classId, false, (int)CellarPropertyTypeFilter.All)
.Where(flid => cache.GetIsCustomField(flid));
foreach (int flid in customFieldIds)
{
string label = LcmMetaData.GetFieldNameOrNull(flid);
if (label == null)
{
continue;
}
string lfCustomFieldName = ConvertUtilities.NormalizedFieldName(label, fieldSourceType);
CellarPropertyType LcmFieldType = (CellarPropertyType)LcmMetaData.GetFieldType(flid);
lfCustomFieldTypes[lfCustomFieldName] = LcmFieldType.ToString();
string lfCustomFieldType;
if (CellarPropertyTypeToLfCustomFieldType.TryGetValue(LcmFieldType, out lfCustomFieldType))
{
// Get custom field configuration info
LfConfigFieldBase fieldConfig = null;
if (lfCustomFieldType.EndsWith("ListRef"))
{
// List references, whether single or multi, need a list code
string listCode = GetParentListCode(flid);
fieldConfig = GetLfCustomFieldOptionListConfig(label, lfCustomFieldType, listCode);
}
else if (lfCustomFieldType == CellarPropertyTypeToLfCustomFieldType[CellarPropertyType.OwningAtomic])
{
// Multiparagraphs don't need writing systems
fieldConfig = GetLfCustomFieldMultiParagraphConfig(label, lfCustomFieldType);
}
else
{
// Single line or MultiText fields need writing systems
int fieldWs = LcmMetaData.GetFieldWs(flid);
// That's a "magic" ws, which we need to expand into a (list of) real writing system(s).
#if FW8_COMPAT
var wsesForThisField = new List <IWritingSystem>();
#else
var wsesForThisField = new List <CoreWritingSystemDefinition>();
#endif
// GetWritingSystemList() in FW 8.3 is buggy and doesn't properly handle the kwsAnal and kwsVern cases, so we handle them here instead.
switch (fieldWs)
{
case WritingSystemServices.kwsAnal:
wsesForThisField.Add(servLoc.LanguageProject.DefaultAnalysisWritingSystem);
break;
case WritingSystemServices.kwsVern:
wsesForThisField.Add(servLoc.LanguageProject.DefaultVernacularWritingSystem);
break;
default:
wsesForThisField = WritingSystemServices.GetWritingSystemList(cache, fieldWs, forceIncludeEnglish: false);
break;
}
#if FW8_COMPAT
IEnumerable <string> inputSystems = wsesForThisField.Select(LcmWs => LcmWs.Id);
#else
IEnumerable <string> inputSystems = wsesForThisField.Select(LcmWs => LcmWs.LanguageTag);
#endif
// GetWritingSystemList returns all analysis WSes even when asked for just one, so if this
// is a single-line custom field, trim the WSes down to just the first one
if (lfCustomFieldType.StartsWith("Single"))
{
inputSystems = inputSystems.Take(1);
}
fieldConfig = GetLfCustomFieldMultiTextConfig(label, lfCustomFieldType, inputSystems.ToList());
}
if (fieldConfig != null)
{
lfCustomFieldConfig[lfCustomFieldName] = fieldConfig;
}
}
}
}
private void RemoveMongoEntriesDeletedInLcm()
{
IEnumerable <LfLexEntry> lfEntries = Connection.GetRecords <LfLexEntry>(LfProject, MagicStrings.LfCollectionNameForLexicon);
foreach (LfLexEntry lfEntry in lfEntries)
{
if (lfEntry.Guid == null)
{
continue;
}
if (!ServiceLocator.ObjectRepository.IsValidObjectId(lfEntry.Guid.Value) ||
!ServiceLocator.ObjectRepository.GetObject(lfEntry.Guid.Value).IsValidObject)
{
if (lfEntry.IsDeleted)
{
// Don't need to delete this record twice
continue;
}
lfEntry.IsDeleted = true;
lfEntry.DateModified = DateTime.UtcNow;
Logger.Info("LcmToMongo: Deleted LfEntry {0} ({1})", lfEntry.Guid, ConvertUtilities.EntryNameForDebugging(lfEntry));
Connection.UpdateRecord(LfProject, lfEntry);
}
}
}
/// <summary>
/// Set custom field data for one field (specified by owner HVO and field ID).
/// </summary>
/// <returns><c>true</c>, if custom field data was set, <c>false</c> otherwise (e.g., if value hadn't changed,
/// or value was null, or field type was one not implemented in LCM, such as CellarPropertyType.Float).</returns>
/// <param name="hvo">HVO of object whose field we're setting.</param>
/// <param name="flid">Field ID of custom field to set.</param>
/// <param name="value">Field's new value (as returned by GetCustomFieldData).</param>
/// <param name="guidOrGuids">GUID or guids associated with new value (as returned by GetCustomFieldData).
/// May be null or BsonNull.Value if no GUIDs associated with this value.</param>
public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue guidOrGuids)
{
if ((value == null) || (value == BsonNull.Value) ||
((value.BsonType == BsonType.Array) && (value.AsBsonArray.Count == 0)))
{
return(false);
}
List <Guid> fieldGuids = new List <Guid>();
if (guidOrGuids == null || guidOrGuids == BsonNull.Value)
{
// Leave fieldGuids as an empty list
}
else
{
if (guidOrGuids is BsonArray)
{
fieldGuids.AddRange(guidOrGuids.AsBsonArray.Select(bsonValue => ParseGuidOrDefault(bsonValue.AsString)));
}
else
{
fieldGuids.Add(ParseGuidOrDefault(guidOrGuids.AsString));
}
}
ISilDataAccessManaged data = (ISilDataAccessManaged)cache.DomainDataByFlid;
CellarPropertyType fieldType = (CellarPropertyType)lcmMetaData.GetFieldType(flid);
string fieldName = lcmMetaData.GetFieldNameOrNull(flid);
// logger.Debug("Custom field named {0} has type {1}", fieldName, fieldType.ToString());
if (fieldName == null)
{
return(false);
}
// Valid field types in LCM are GenDate, Integer, String, OwningAtomic, ReferenceAtomic, and ReferenceCollection, so that's all we implement.
switch (fieldType)
{
case CellarPropertyType.GenDate:
{
var valueAsMultiText = BsonSerializer.Deserialize <LfMultiText>(value.AsBsonDocument);
string valueAsString = valueAsMultiText.BestString(new string[] { MagicStrings.LanguageCodeForGenDateFields });
if (string.IsNullOrEmpty(valueAsString))
{
return(false);
}
GenDate valueAsGenDate;
if (GenDate.TryParse(valueAsString, out valueAsGenDate))
{
GenDate oldValue = data.get_GenDateProp(hvo, flid);
if (oldValue == valueAsGenDate)
{
return(false);
}
else
{
data.SetGenDate(hvo, flid, valueAsGenDate);
return(true);
}
}
return(false);
}
case CellarPropertyType.Integer:
{
var valueAsMultiText = BsonSerializer.Deserialize <LfMultiText>(value.AsBsonDocument);
string valueAsString = valueAsMultiText.BestString(new string[] { MagicStrings.LanguageCodeForIntFields });
if (string.IsNullOrEmpty(valueAsString))
{
return(false);
}
int valueAsInt;
if (int.TryParse(valueAsString, out valueAsInt))
{
int oldValue = data.get_IntProp(hvo, flid);
if (oldValue == valueAsInt)
{
return(false);
}
else
{
data.SetInt(hvo, flid, valueAsInt);
return(true);
}
}
return(false);
}
case CellarPropertyType.OwningAtomic:
{
// Custom field is a MultiparagraphText, which is an IStText object in LCM
IStTextRepository textRepo = servLoc.GetInstance <IStTextRepository>();
Guid fieldGuid = fieldGuids.FirstOrDefault();
IStText text;
if (!textRepo.TryGetObject(fieldGuid, out text))
{
int currentFieldContentsHvo = data.get_ObjectProp(hvo, flid);
if (currentFieldContentsHvo != LcmCache.kNullHvo)
{
text = (IStText)cache.GetAtomicPropObject(currentFieldContentsHvo);
}
else
{
// NOTE: I don't like the "magic" -2 number below, but LCM doesn't seem to have an enum for this. 2015-11 RM
int newStTextHvo = data.MakeNewObject(cache.GetDestinationClass(flid), hvo, flid, -2);
text = (IStText)cache.GetAtomicPropObject(newStTextHvo);
}
}
// Shortcut: if text contents haven't changed, we don't want to change anything at all
BsonValue currentLcmTextContents = ConvertUtilities.GetCustomStTextValues(text, flid,
servLoc.WritingSystemManager, lcmMetaData, cache.DefaultUserWs);
if ((currentLcmTextContents == BsonNull.Value || currentLcmTextContents == null) &&
(value == BsonNull.Value || value == null))
{
return(false);
}
if (currentLcmTextContents != null && currentLcmTextContents.Equals(value))
{
// No changes needed.
return(false);
}
// BsonDocument passed in contains "paragraphs". ParseCustomStTextValuesFromBson wants only a "value" element
// inside the doc, so we'll need to construct a new doc for the StTextValues.
BsonDocument doc = value.AsBsonDocument;
LfMultiParagraph multiPara = BsonSerializer.Deserialize <LfMultiParagraph>(doc);
// Now we have another way to check for "old value and new value were the same": if the LCM multiparagraph was empty,
// GetCustomStTextValues will have returned null -- so if this multiPara has no paragraphs, that's also an unchanged situation
if ((multiPara.Paragraphs == null || multiPara.Paragraphs.Count <= 0) &&
(currentLcmTextContents == BsonNull.Value || currentLcmTextContents == null))
{
return(false);
}
int wsId;
if (multiPara.InputSystem == null)
{
wsId = lcmMetaData.GetFieldWs(flid);
}
else
{
wsId = servLoc.WritingSystemFactory.GetWsFromStr(multiPara.InputSystem);
}
ConvertUtilities.SetCustomStTextValues(text, multiPara.Paragraphs, wsId);
return(true);
}
case CellarPropertyType.ReferenceAtomic:
if (fieldGuids.FirstOrDefault() != Guid.Empty)
{
int referencedHvo = data.get_ObjFromGuid(fieldGuids.FirstOrDefault());
int oldHvo = data.get_ObjectProp(hvo, flid);
if (referencedHvo == oldHvo)
{
return(false);
}
else
{
data.SetObjProp(hvo, flid, referencedHvo);
// TODO: What if the value of the referenced object has changed in LanguageForge? (E.g., change that possibility's text from "foo" to "bar")
// Need to implement that scenario.
return(true);
}
}
else
{
// It's a reference to an ICmPossibility instance: create a new entry in appropriate PossibilityList
LfStringField valueAsLfStringField = BsonSerializer.Deserialize <LfStringField>(value.AsBsonDocument);
string nameHierarchy = valueAsLfStringField.Value;
if (nameHierarchy == null)
{
return(false);
}
int fieldWs = lcmMetaData.GetFieldWs(flid);
// Oddly, this can return 0 for some custom fields. TODO: Find out why: that seems like it would be an error.
if (fieldWs == 0)
{
fieldWs = cache.DefaultUserWs; // TODO: Investigate, because this should probably be wsEn instead so that we can create correct keys.
}
if (fieldWs < 0)
{
// FindOrCreatePossibility has a bug where it doesn't handle "magic" writing systems (e.g., -1 for default analysis, etc) and
// throws an exception instead. So we need to get a real ws here.
fieldWs = WritingSystemServices.ActualWs(cache, fieldWs, hvo, flid);
}
ICmPossibilityList parentList = GetParentListForField(flid);
ICmPossibility newPoss = parentList.FindOrCreatePossibility(nameHierarchy, fieldWs);
int oldHvo = data.get_ObjectProp(hvo, flid);
if (newPoss.Hvo == oldHvo)
{
return(false);
}
else
{
data.SetObjProp(hvo, flid, newPoss.Hvo);
return(true);
}
}
case CellarPropertyType.ReferenceCollection:
case CellarPropertyType.ReferenceSequence:
{
if (value == null || value == BsonNull.Value)
{
// Can't write null to a collection or sequence in LCM; it's forbidden. So data.SetObjProp(hvo, flid, LcmCache.kNullHvo) will not work.
// Instead, we delete all items from the existing collection or sequence, and thus store an empty coll/seq in LCM.
int oldSize = data.get_VecSize(hvo, flid);
if (oldSize == 0)
{
// It was already empty, so leave it unchanged so we don't cause unnecessary changes in the .fwdata XML (and unnecessary Mercurial commits).
return(false);
}
else
{
data.Replace(hvo, flid, 0, oldSize, null, 0); // This is how you set an empty array
return(true);
}
}
int fieldWs = lcmMetaData.GetFieldWs(flid);
// TODO: Investigate why this is sometimes coming back as 0 instead of as a real writing system ID
if (fieldWs == 0)
{
fieldWs = cache.DefaultUserWs;
}
ICmPossibilityList parentList = GetParentListForField(flid);
LfStringArrayField valueAsStringArray = BsonSerializer.Deserialize <LfStringArrayField>(value.AsBsonDocument);
// Step 1: Get ICmPossibility instances from the string keys that LF gave us
// First go through all the GUIDs we have and match them up to the keys. If they match up,
// then remove the keys from the list. Any remaining keys get looked up with FindOrCreatePossibility(), so now we
// have a complete set of GUIDs (or ICmPossibility objects, which works out to the same thing).
// TODO: This is all kind of ugly, and WAY too long for one screen. I could put it in its own function,
// but there's really no real gain from that, as it simply moves the logic even further away from where
// it needs to be. There's not really a *good* way to achieve simplicity with this code design, unfortunately.
// The only thing that would be close to simple would be to call some functions from the LcmToMongo option list
// converters, and that's pulling in code from the "wrong" direction, which has its own ugliness. Ugh.
HashSet <string> keysFromLF = new HashSet <string>(valueAsStringArray.Values);
var fieldObjs = new List <ICmPossibility>();
foreach (Guid guid in fieldGuids)
{
ICmPossibility poss;
string key = "";
if (guid != default(Guid))
{
poss = servLoc.GetInstance <ICmPossibilityRepository>().GetObject(guid);
if (poss == null)
{
// TODO: Decide what to do with possibilities deleted from LCM
key = "";
}
else
{
if (poss.Abbreviation == null)
{
key = "";
}
else
{
ITsString keyTss = poss.Abbreviation.get_String(wsEn);
key = keyTss == null ? "" : keyTss.Text ?? "";
}
fieldObjs.Add(poss);
}
}
keysFromLF.Remove(key);
// Ignoring return value (HashSet.Remove returns false if the key wasn't present), because false could mean one of two things:
// 1. The CmPossibility had its English abbreviation changed in LCM, but LF doesn't know this yet.
// If this is the case, the LF key won't match, but the GUID will still match. So we might end up creating
// duplicate entries below with the FindOrCreatePossibility. TODO: Need to verify that LCM->LF possibility lists
// get updated correctly if renames happen! (... Or use the OptionList converters, that's what they were for.)
// 2. The CmPossibility was just created in LF and isn't in LCM yet. In which case we should have been using the
// OptionList converters, which would hopefully have handled creating the ICmPossibility instane in LCM.
// Either way, we can't really use that fact later, since we can't be certain if the possibility was renamed or created.
}
// Any remaining keysFromLF strings did not have corresponding GUIDs in Mongo.
// This is most likely because they were added by LF, which doesn't write to the customFieldGuids field.
// So we assume they exist in FW, and just look them up.
foreach (string key in keysFromLF)
{
ICmPossibility poss = parentList.FindOrCreatePossibility(key, wsEn);
// TODO: If this is a new possibility, then we need to populate it with ALL the corresponding data from LF,
// which we don't necessarily have at this point. Need to make that a separate step in the Send/Receive: converting option lists first.
fieldObjs.Add(poss);
}
// logger.Debug("Custom field {0} for CmObject {1}: BSON list was [{2}] and customFieldGuids was [{3}]. This was translated to keysFromLF = [{4}] and fieldObjs = [{5}]",
// fieldName,
// hvo,
// String.Join(", ", valueAsStringArray.Values),
// String.Join(", ", fieldGuids.Select(g => g.ToString())),
// String.Join(", ", keysFromLF.AsEnumerable()),
// String.Join(", ", fieldObjs.Select(poss => poss.AbbrAndName))
// );
// Step 2: Remove any objects from the "old" list that weren't in the "new" list
// We have to look them up by HVO because that's the only public API available in LCM
// Following logic inspired by XmlImportData.CopyCustomFieldData in FieldWorks source
int[] oldHvosArray = data.VecProp(hvo, flid);
int[] newHvosArray = fieldObjs.Select(poss => poss.Hvo).ToArray();
// Shortcut check
if (oldHvosArray.SequenceEqual(newHvosArray))
{
// Nothing to do, so return now so that we don't cause unnecessary changes and commits in Mercurial
return(false);
}
HashSet <int> newHvos = new HashSet <int>(newHvosArray);
HashSet <int> combinedHvos = new HashSet <int>();
// Loop backwards so deleting items won't mess up indices of subsequent deletions
for (int idx = oldHvosArray.Length - 1; idx >= 0; idx--)
{
int oldHvo = oldHvosArray[idx];
if (newHvos.Contains(oldHvo))
{
combinedHvos.Add(oldHvo);
}
else
{
data.Replace(hvo, flid, idx, idx + 1, null, 0); // Important to pass *both* null *and* 0 here to remove items
}
}
// Step 3: Add any objects from the "new" list that weren't in the "old" list
foreach (int newHvo in newHvosArray)
{
if (combinedHvos.Contains(newHvo))
{
continue;
}
// This item was added in the new list
data.Replace(hvo, flid, combinedHvos.Count, combinedHvos.Count, new int[] { newHvo }, 1);
combinedHvos.Add(newHvo);
}
return(true);
}
case CellarPropertyType.String:
{
var valueAsMultiText = BsonSerializer.Deserialize <LfMultiText>(value.AsBsonDocument);
int wsIdForField = lcmMetaData.GetFieldWs(flid);
string wsStrForField = servLoc.WritingSystemFactory.GetStrFromWs(wsIdForField);
KeyValuePair <string, string> kv = valueAsMultiText.BestStringAndWs(new string[] { wsStrForField });
string foundWs = kv.Key ?? string.Empty;
string foundData = kv.Value ?? string.Empty;
int foundWsId = servLoc.WritingSystemFactory.GetWsFromStr(foundWs);
if (foundWsId == 0)
{
return(false); // Skip any unidentified writing systems
}
ITsString oldValue = data.get_StringProp(hvo, flid);
ITsString newValue = ConvertMongoToLcmTsStrings.SpanStrToTsString(foundData, foundWsId, servLoc.WritingSystemFactory);
if (oldValue != null && TsStringUtils.GetDiffsInTsStrings(oldValue, newValue) == null) // GetDiffsInTsStrings() returns null when there are no changes
{
return(false);
}
else
{
data.SetString(hvo, flid, newValue);
return(true);
}
}
default:
return(false);
// TODO: Maybe issue a proper warning (or error) log message for "field type not recognized"?
}
}
public void RunConversion()
{
Logger.Notice("LcmToMongo: Converting lexicon for project {0}", LfProject.ProjectCode);
ILexEntryRepository repo = GetInstance <ILexEntryRepository>();
if (repo == null)
{
Logger.Error("Can't find LexEntry repository for FieldWorks project {0}", LfProject.ProjectCode);
return;
}
// Custom field configuration AND view configuration should all be set at once
Dictionary <string, LfConfigFieldBase> lfCustomFieldList = new Dictionary <string, LfConfigFieldBase>();
Dictionary <string, string> lfCustomFieldTypes = new Dictionary <string, string>();
_convertCustomField.WriteCustomFieldConfig(lfCustomFieldList, lfCustomFieldTypes);
Connection.SetCustomFieldConfig(LfProject, lfCustomFieldList, lfCustomFieldTypes);
Dictionary <Guid, DateTime> previousModificationDates = Connection.GetAllModifiedDatesForEntries(LfProject);
int i = 1;
foreach (ILexEntry LcmEntry in repo.AllInstances())
{
bool createdEntry = false;
DateTime previousDateModified;
if (!previousModificationDates.TryGetValue(LcmEntry.Guid, out previousDateModified))
{
// Looks like this entry is new in Lcm
createdEntry = true;
previousDateModified = DateTime.MinValue; // Ensure it will seem modified when comparing it later
}
// Remember that Lcm's DateModified is stored in local time for some incomprehensible reason...
if (!createdEntry && previousDateModified.ToLocalTime() == LcmEntry.DateModified)
{
// Hasn't been modified since last time: just skip this record entirely
continue;
}
LfLexEntry lfEntry = LcmLexEntryToLfLexEntry(LcmEntry);
lfEntry.IsDeleted = false;
Logger.Info("{3} - LcmToMongo: {0} LfEntry {1} ({2})", createdEntry ? "Created" : "Modified", lfEntry.Guid, ConvertUtilities.EntryNameForDebugging(lfEntry), i++);
Connection.UpdateRecord(LfProject, lfEntry);
}
LfProject.IsInitialClone = false;
RemoveMongoEntriesDeletedInLcm();
// Logger.Debug("Running FtMComments, should see comments show up below:");
var commCvtr = new ConvertLcmToMongoComments(Connection, LfProject, Logger, Progress, ProjectRecordFactory);
commCvtr.RunConversion();
}