public async Task RequestExtract(CancellationToken token, string id, ExtractType type, string extractorPath)
{
try
{
string extracttype = type switch
{
ExtractType.Full => "complete",
ExtractType.Incremental => "incremental",
_ => "incremental"
};
string request_string = $"{_settings.GisServerUrl}?request=export&export_type={extracttype}&path={Uri.EscapeDataString(extractorPath)}&correlation_id={id}";
var method = new HttpMethod("GET");
var request = new HttpRequestMessage(method, request_string);
var res = await client.SendAsync(request);
var message = await res.Content.ReadAsStringAsync();
XDocument doc = XDocument.Parse(message);
if (res.StatusCode == System.Net.HttpStatusCode.OK && doc?.Element("return")?.Element("service_response")?.Element("status")?.Value == "OK")
{
await CheckForIdfOutput(extractorPath, token);
}
else
{
throw new ExtractorException($"Could not trigger an extract. Message from gis server:{message}");
}
}
catch (Exception ex)
{
throw new ExtractorException("Exception occured during extraction, see inner exception for details", ex);
}
}
private bool CheckState(Entity e1, Entity e2, ExtractType state, bool checkKeys)
{
List <string> keys = null;
List <string> foreignKeys = null;
var stateE1 = e1.ExtractState(state);
var stateE2 = e2.ExtractState(state);
if (stateE1.Count != stateE2.Count)
{
return(false);
}
if (checkKeys == false)
{
keys = GetKey(e1);
foreignKeys = GetForeignKeys(e1);
}
var zip = stateE1.Zip(stateE2, (a, b) => new { name = a.Key, a = a.Value, b = b.Value });
foreach (var v in zip)
{
if (v.a != v.b && v.a != null && v.a.Equals(v.b) == false)
{
if (checkKeys == false)
{
if (keys.Contains(v.name) || foreignKeys.Contains(v.name))
{
continue;
}
}
return(false);
}
}
return(true);
}
public static string FindDetails(List <EntityRecord> entityRecords, ExtractType extractType)
{
string value = "";
switch (extractType)
{
case ExtractType.Name:
string name = entityRecords.Where(a => a.Type == "Person").FirstOrDefault()?.Name;
if (name != null)
{
value = name.Substring(0, name.Length > 20 ? 20 : name.Length);
}
break;
case ExtractType.Email:
value = entityRecords.Where(a => a.Type == "Email" && a.WikipediaUrl == null).FirstOrDefault()?.Name;
break;
case ExtractType.Phone:
List <string> numberList = entityRecords.Where(a => a.Type == "Quantity" && a.SubType == "Number").Select(a => a.Name).ToList();
value = numberList.Where(a => a.Length >= 10).FirstOrDefault();
break;
case ExtractType.LinkedIn:
List <string> urlList = entityRecords.Where(a => a.Type == "URL").Select(a => a.Name).ToList();
value = urlList.Where(a => a.Contains("linkedin")).FirstOrDefault();
break;
}
return(value);
}
private string GetPumpClass(string s)
{
ExtractType et = ImpExtractType.getInstance();
string type = et.ExtractTypeFromName(s);
return(type);
}
/// <summary>
/// Indexer Component to use for indexing
/// </summary>
/// <param name="splitter">Splitter to be used</param>
/// <param name="extractType">Split Type: Comment / Source code</param>
/// <param name="dictionary">Dictionary to be used. Is safe to share among processes.</param>
/// <param name="textExtractors">Text extractors to be used. Is safe to share among processes.</param>
/// <param name="textCorrector">Text Corrector to use</param>
/// <param name="stemmer">Stemmer to be used.</param>
/// <param name="filesToScan">Files to scan.</param>
/// <param name="notificationHandler">Notification handler. Use null to not receive any notification. Default action will be used in case of exceptions. Is safe to share among processes.</param>
private IndexerConfiguration(ISplitter splitter, ExtractType extractType,
IDictionary dictionary, IList <ITextExtractor> textExtractors, ITextCorrector textCorrector, IStemmer stemmer,
List <IndexerFile> filesToScan, INotificationHandler notificationHandler = null)
{
if (dictionary == null)
{
throw new ArgumentNullException(nameof(dictionary));
}
if (splitter == null)
{
throw new ArgumentNullException(nameof(splitter), "Splitter cannot be null. Try setting CustomSplitter or set UseDefaultSplitterOfType to some SplitterType.");
}
Splitter = splitter;
ExtractType = extractType;
Dictionary = dictionary;
TextExtractors = textExtractors;
TextCorrector = textCorrector;
Stemmer = stemmer;
FilesToScan = filesToScan;
NotificationHandler = notificationHandler;
}
public List <Person> ExtractPeopleFromMovieObject(Movie movie, ExtractType type)
{
List <Entities.Person> people;
switch (type)
{
case ExtractType.Actor:
people = movie.Actors.Select(x => new Person {
Name = x.ActorName
}).ToList();
break;
case ExtractType.Director:
people = movie.Directors.Select(x => new Person {
Name = x.Name
}).ToList();
break;
case ExtractType.Writer:
people = movie.Writers.Select(x => new Person {
Name = x.Name
}).ToList();
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
return(people);
}
public void UpdateSendStatus(ExtractType extractType, List <SentItem> sentItems)
{
try
{
switch (extractType)
{
case ExtractType.Patient:
_patientMnchExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.AncVisit:
_ancVisitExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.CwcEnrollment:
_cwcEnrolmentExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.CwcVisit:
_cwcVisitExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.Hei:
_heiExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.MatVisit:
_matVisitExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.MnchArt:
_mnchArtExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.MnchEnrollment:
_mnchEnrolmentExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.MnchLab:
_mnchLabExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.MotherBabyPair:
_motherBabyPairExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PncVisit:
_pncVisitExtractRepository.UpdateSendStatus(sentItems);
break;
}
}
catch (Exception e)
{
Log.Error(e, "Sent status");
}
}
public Extract(ExtractType type, string description, DateTime date, int credits, Operation operation, int balance, int clientId)
{
Type = type;
Description = description;
Date = date;
Credits = credits;
Operation = operation;
Balance = balance;
ClientId = clientId;
}
public static string GetPumpClassFromPumpName(string s)
{
string result = "";
ExtractType et = ImpExtractType.getInstance();
string type = et.ExtractTypeFromName(s);
result = Util.Comm.GetPumpClass(type);
return(result);
}
/// <summary>
/// 返回所有匹配的数据
/// </summary>
/// <typeparam name="T">BaseParm的继承类 用于配置非通用化的参数</typeparam>
/// <param name="extractType">匹配方式 枚举值</param>
/// <param name="Txt">待匹配的文本</param>
/// <param name="path">正则、Xpath、Jpath表达式</param>
/// <param name="parm">非通用化参数 例如正则的分组 Xpath的属性等</param>
/// <returns>所有匹配的数据</returns>
public static List <string> GetResults <T>(ExtractType extractType, string Txt, string path, T parm) where T : BaseParm
{
var et = typesdic[extractType];
if (et != null)
{
return(((IExtract <T>)Activator.CreateInstance(et.MakeGenericType(typeof(T)))).GetResults(Txt, path, parm));
}
return(new List <String>());
}
/// <summary>
/// 校验
/// </summary>
/// <typeparam name="T">BaseParm的继承类 用于配置非通用化的参数</typeparam>
/// <param name="extractType">匹配方式 枚举值</param>
/// <param name="Txt">待匹配的文本</param>
/// <param name="path">正则、Xpath、Jpath表达式</param>
/// <param name="parm">非通用化参数 例如正则的分组、Xpath的属性、需要校验的值等</param>
/// <returns>校验结果</returns>
public static bool Check <T>(ExtractType extractType, string Txt, string path, T parm) where T : BaseParm
{
var et = typesdic[extractType];
if (et != null)
{
return(((IExtract <T>)Activator.CreateInstance(et.MakeGenericType(typeof(T)))).Check(Txt, path, parm));
}
return(false);;
}
/// <summary>
/// 匹配第一条 不带扩展参数
/// </summary>
/// <param name="extractType">匹配方式 枚举值</param>
/// <param name="Txt">待匹配的文本</param>
/// <param name="path">正则、Xpath、Jpath表达式</param>
/// <returns匹配到的第一条数据></returns>
public static string GetResult(ExtractType extractType, string Txt, string path)
{
var et = typesdic[extractType];
if (et != null)
{
return(((IExtract <BaseParm>)Activator.CreateInstance(et.MakeGenericType(typeof(BaseParm)))).GetResult(Txt, path, null));
}
return(String.Empty);;
}
public List<Movie> ExtractMoviesFromPersonObject(Person person, ExtractType type)
{
var filter = (type == ExtractType.Actor
? "Actor"
: (type == ExtractType.Director
? "Director" : "Writer"));
return
person.Filmographies.Where(x => x.Section == filter)
.SelectMany(
f => f.Filmography.Select(film => new Movie {Title = film.Title, Year = film.Year}).ToList())
.ToList();
}
/// <summary>
/// 静态类进行反射装载
/// </summary>
static ExtractMethod()
{
var types = Assembly.GetExecutingAssembly().DefinedTypes.Where(dt => dt.IsClass && dt.IsGenericTypeDefinition && Array.Exists(dt.GetInterfaces(), t => t.GetGenericTypeDefinition() == typeof(IExtract <>)));
types?.ToList().ForEach(item => {
ExtractType et = (item.GetCustomAttribute(typeof(ExtractTypeAttribute)) as ExtractTypeAttribute).extracttype;
if (et != ExtractType.None && !typesdic.ContainsKey(et))
{
typesdic.Add(et, item);
}
});
}
/// <summary>
/// 替换符合匹配条件的数据 目前只支持正则
/// </summary>
/// <param name="extractType">匹配方式 枚举值</param>
/// <param name="Txt">待匹配的文本</param>
/// <param name="path">正则表达式</param>
/// <param name="RepStr">替换的文本</param>
/// <returns>替换后的字符串</returns>
public static string Replace(string Txt, string path, string RepStr, ExtractType extractType = ExtractType.Regex)
{
if (extractType == ExtractType.Regex)
{
var et = typesdic[extractType];
if (et != null)
{
return(((IExtract <BaseParm>)Activator.CreateInstance(et.MakeGenericType(typeof(BaseParm)))).Replace(Txt, path, RepStr));
}
}
return(Txt);
}
/// <summary>
/// 使用正则表达式进行分割
/// </summary>
/// <param name="extractType">匹配方式 枚举值</param>
/// <param name="Txt">待匹配的文本</param>
/// <param name="path">正则表达式</param>
/// <returns>分割后的字符串集合</returns>
public static List <string> Split(string Txt, string path, ExtractType extractType = ExtractType.Regex)
{
List <String> list = new List <string>();
if (extractType == ExtractType.Regex)
{
var et = typesdic[extractType];
if (et != null)
{
return(((IExtract <BaseParm>)Activator.CreateInstance(et.MakeGenericType(typeof(BaseParm)))).Split(Txt, path));
}
}
return(list);
}
/// <summary>
/// Runs indexer
/// </summary>
private void BackgroundWorker_DoWork(object sender, DoWorkEventArgs e)
{
IsIndexingInProgress = true;
UpdateCanStartIndexing();
IndexerConfiguration.SplitterSetting splitterSetting = new IndexerConfiguration.SplitterSetting()
{
UseDefaultSplitterOfType = ApplyBestSuffixSplit
? SplitterType.Advance
: (ApplyCamelCaseSplit ? SplitterType.CamelCase : SplitterType.None),
};
IndexerConfiguration.DictionarySetting dictionarySetting = new IndexerConfiguration.DictionarySetting()
{
DefaultDictionaryRemoveEnglishStopWord = RemoveEnglishStopWords,
DefaultDictionaryRemoveProgrammingStopWord = RemoveProgrammingStopWords
};
IndexerConfiguration.TextCorrectorSetting textCorrectorSetting = new IndexerConfiguration.
TextCorrectorSetting()
{
UseDefaultSpellChecker = ApplySpellCheck
};
ExtractType extractType = 0;
SplitTypes.Where(x => x.IsSelected).ToList().ForEach(x => extractType |= x.Value);
IndexerConfiguration.TextExtractorSetting textExtractorSetting = new IndexerConfiguration.
TextExtractorSetting()
{
ExtractType = extractType
};
IStemmer stemmer = ApplyLemmatizer ? _stanfordLemmatizer : null;
IndexerConfiguration.StemmerSetting stemmerSetting = new IndexerConfiguration.StemmerSetting()
{
CustomStemmer = stemmer,
UseDefaultPorterStemmer = ApplyStemmer
};
List <IndexerFile> selectedFiles = SelectedFiles.ToList();
IndexerConfiguration indexerConfiguration = IndexerConfiguration.GetIndexerConfiguration(splitterSetting, dictionarySetting, textExtractorSetting, textCorrectorSetting, stemmerSetting, selectedFiles, this);
Indexer indexer = new Indexer(indexerConfiguration);
e.Result = indexer.Execute();
}
public List <Movie> ExtractMoviesFromPersonObject(Person person, ExtractType type)
{
var filter = (type == ExtractType.Actor
? "Actor"
: (type == ExtractType.Director
? "Director" : "Writer"));
return
(person.Filmographies.Where(x => x.Section == filter)
.SelectMany(
f => f.Filmography.Select(film => new Movie {
Title = film.Title, Year = film.Year
}).ToList())
.ToList());
}
/// <summary>
/// Extract comments and string literals and/or identifiers from source code
/// </summary>
/// <param name="text">Source code in which the extraction is done</param>
/// <param name="extractType">Identifies split only comments/String literals, only identifiers or both of type <see cref="ExtractType"/></param>
/// <returns>List of strings</returns>
public IEnumerable <string> Extract(string text, ExtractType extractType)
{
if ((extractType & ExtractType.Comments) == ExtractType.Comments)
{
foreach (string item in ExtractComments(text))
{
yield return(item);
}
}
if ((extractType & ExtractType.IdentifiersAndStringLiterals) == ExtractType.IdentifiersAndStringLiterals)
{
foreach (string item in ExtractIdentifiersAndStringLiterals(text))
{
yield return(item);
}
}
}
public void UpdateSendStatus(ExtractType extractType, List <SentItem> sentItems)
{
try
{
switch (extractType)
{
case ExtractType.Patient:
_patientExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientArt:
_artExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientBaseline:
_baselinesExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientLab:
_laboratoryExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientPharmacy:
_pharmacyExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientStatus:
_statusExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientVisit:
_visitExtractRepository.UpdateSendStatus(sentItems);
break;
case ExtractType.PatientAdverseEvent:
_adverseEventExtractRepository.UpdateSendStatus(sentItems);
break;
}
}
catch (Exception e)
{
Log.Error(e, "Sent status");
}
}
public void UpdateSendStatus(ExtractType extractType, List <SentItem> sentItems)
{
try
{
if (extractType == ExtractType.Patient)
{
_patientExtractRepository.UpdateSendStatus(sentItems);
}
if (extractType == ExtractType.PatientArt)
{
_artExtractRepository.UpdateSendStatus(sentItems);
}
}
catch (Exception e)
{
Log.Error(e, "Sent status");
}
}
public List<Person> ExtractPeopleFromMovieObject(Movie movie, ExtractType type)
{
List<Entities.Person> people;
switch (type)
{
case ExtractType.Actor:
people = movie.Actors.Select(x => new Person {Name = x.ActorName}).ToList();
break;
case ExtractType.Director:
people = movie.Directors.Select(x => new Person {Name = x.Name}).ToList();
break;
case ExtractType.Writer:
people = movie.Writers.Select(x => new Person {Name = x.Name}).ToList();
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
return people;
}
public static IDictionary <string, object> ExtractState(this Entity entity, ExtractType extractType)
{
Entity extractEntity;
if (extractType == ExtractType.OriginalState && entity.HasChanges)
{
extractEntity = entity.GetOriginal();
}
else
{
extractEntity = entity;
}
Dictionary <string, object> returnDictionary = new Dictionary <string, object>();
foreach (PropertyInfo currentPropertyInfo in GetDataMembers(extractEntity))
{
//if(currentPropertyInfo.IsDefined(typeof(KeyAttribute), true))
// continue;
object currentObject = currentPropertyInfo.GetValue(extractEntity, null);
returnDictionary[currentPropertyInfo.Name] = currentObject;
}
return(returnDictionary);
}
/// <summary>
/// Import variables into the current variables table from an array.
/// </summary>
/// <param name="ctx">Runtime context.</param>
/// <param name="locals">The table of defined variables.</param>
/// <param name="vars">The <see cref="PhpArray"/> containing names of variables and values to be assigned to them.</param>
/// <param name="type">The type of the extraction.</param>
/// <param name="prefix">The prefix (can be a <B>null</B> reference) of variables names.</param>
/// <returns>The number of variables actually affected by the extraction.</returns>
/// <exception cref="PhpException"><paramref name="type"/> is invalid.</exception>
/// <exception cref="PhpException"><paramref name="vars"/> is a <B>null</B> reference.</exception>
/// <exception cref="InvalidCastException">Some key of <paramref name="locals"/> is not type of <see cref="string"/>.</exception>
public static int extract(Context ctx, [ImportLocals] PhpArray locals, PhpArray vars, ExtractType type = ExtractType.Overwrite, string prefix = null)
{
if (vars == null)
{
//PhpException.ArgumentNull("vars");
//return 0;
throw new ArgumentNullException(nameof(vars));
}
if (vars.Count == 0)
{
return(0);
}
// unfortunately, type contains flags are combined with enumeration:
bool refs = (type & ExtractType.Refs) != 0;
type &= ExtractType.NonFlags;
//
//
//
int extracted_count = 0;
using (var enumerator = vars.GetFastEnumerator())
while (enumerator.MoveNext())
{
var name = enumerator.CurrentKey.ToString();
if (string.IsNullOrEmpty(name) && type != ExtractType.PrefixInvalid)
{
continue;
}
switch (type)
{
case ExtractType.Overwrite:
// anything is overwritten:
break;
case ExtractType.Skip:
// skips existing name:
if (locals.ContainsKey(name))
{
continue;
}
break;
case ExtractType.IfExists:
// skips nonexistent name:
if (!locals.ContainsKey(name))
{
continue;
}
break;
case ExtractType.PrefixAll:
// prefix anything:
name = string.Concat(prefix, "_", name);
break;
case ExtractType.PrefixInvalid:
// prefixes invalid, others are overwritten:
if (!PhpVariable.IsValidName(name))
{
name = string.Concat(prefix, "_", name);
}
break;
case ExtractType.PrefixSame:
// prefixes existing, others are overwritten:
if (locals.ContainsKey(name))
{
name = string.Concat(prefix, "_", name);
}
break;
case ExtractType.PrefixIfExists:
// prefixes existing, others are skipped:
if (locals.ContainsKey(name))
{
name = string.Concat(prefix, "_", name);
}
else
{
continue;
}
break;
default:
throw new ArgumentException(nameof(type));
//PhpException.InvalidArgument("type", LibResources.GetString("arg_invalid_value"));
//return 0;
}
// invalid names are skipped:
if (PhpVariable.IsValidName(name))
{
// write the value to locals:
if (refs)
{
// makes a reference and writes it back (deep copy is not necessary, "no duplicate pointers" rule preserved):
locals.SetItemAlias(new IntStringKey(name), enumerator.CurrentValueAliased);
}
else
{
// deep copy the value and write into locals
locals.SetItemValue(new IntStringKey(name), enumerator.CurrentValue.GetValue().DeepCopy());
}
extracted_count++;
}
}
//
return(extracted_count);
}
public void GivenAnExtractTypeForACertainUfedFile(string extractTypeDescription)
{
_currentExtractType = (ExtractType)Enum.Parse(typeof(ExtractType), extractTypeDescription);
_currentExtractTypeName = Enum.GetName(typeof(ExtractType), _currentExtractType);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var result = new List <ExtractResult>();
var matchSource = new Dictionary <Match, TypeTag>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
// In ExperimentalMode, AmbiguousFraction like "30000 in 2009" needs to be skipped
if ((Options & NumberOptions.ExperimentalMode) != 0 && AmbiguousFractionConnectorsRegex.Match(m.Value).Success)
{
continue;
}
// In EnablePreview, cases like "last", "next" should not be skipped
if ((Options & NumberOptions.EnablePreview) == 0 && IsRelativeOrdinal(m.Value))
{
continue;
}
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Fliter out cases like "first two", "last one"
// only support in English now
if (ExtractType.Contains(Constants.MODEL_ORDINAL) && RelativeOrdinalFilterRegex != null && RelativeOrdinalFilterRegex.IsMatch(source))
{
continue;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Index == start && o.Length == length))
{
var type = matchSource.Where(p => p.Key.Index == start && p.Key.Length == length)
.Select(p => (p.Value.Priority, p.Value.Name)).Min().Item2;
// Extract negative numbers
if (NegativeNumberTermsRegex != null)
{
var match = NegativeNumberTermsRegex.Match(source.Substring(0, start));
if (match.Success)
{
start = match.Index;
length = length + match.Length;
substr = match.Value + substr;
}
}
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = type,
};
// Add Metadata information for Ordinal
if (ExtractType.Contains(Constants.MODEL_ORDINAL))
{
er.Metadata = new Metadata();
if (IsRelativeOrdinal(substr))
{
er.Metadata.IsOrdinalRelative = true;
}
}
result.Add(er);
}
}
}
else
{
last = i;
}
}
result = FilterAmbiguity(result, source);
return(result);
}
public void UpdateExtractSent(ExtractType extractType, List <Guid> sentIds)
{
DomainEvents.Dispatch(new DwhExtractSentEvent(extractType, sentIds, SendStatus.Sent));
}
public SentItem(Guid id, SendStatus status, string statusInfo, ExtractType extractType) : this(id, status)
{
ExtractType = extractType;
StatusInfo = statusInfo;
}
public CTExtractSentEvent(List <Guid> sentIds, SendStatus status, ExtractType extractType, string statusInfo = "")
{
SentItems = sentIds.Select(x => new SentItem(x, status, statusInfo, extractType)).ToList();
}
public static int Extract(Dictionary<string, object> localVariables, PhpArray/*!*/ vars, ExtractType type,
string prefix)
{
if (vars == null)
{
PhpException.ArgumentNull("vars");
return 0;
}
// unfortunately, type contains flags are combined with enumeration:
bool refs = (type & ExtractType.Refs) != 0;
type &= ExtractType.NonFlags;
//
// construct the action used to set the variable into the locals/globals
//
Action<string/*name*/, object/*value*/> updateVariableFn; // function that writes the value to locals/globals
Predicate<string/*name*/> containsFn; // function that checks if variable exists
PhpArray globals = (localVariables != null) ? null : ScriptContext.CurrentContext.GlobalVariables;
#region select function that writes the variable
if (refs)
{
// makes a reference and writes it back (deep copy is not necessary, "no duplicate pointers" rule preserved):
if (localVariables != null)
{
updateVariableFn = (name, value) =>
{
localVariables[name] = vars[name] = PhpVariable.MakeReference(value);
};
}
else
{
updateVariableFn = (name, value) =>
{
globals[name] = vars[name] = PhpVariable.MakeReference(value);
};
}
}
else
{
if (localVariables != null)
{
updateVariableFn = (name, value) =>
{
// deep copy the value
value = PhpVariable.DeepCopy(PhpVariable.Dereference(value));
// put into locals
object item;
PhpReference ref_item;
if (localVariables.TryGetValue(name, out item) && (ref_item = item as PhpReference) != null)
ref_item.Value = value;
else
localVariables[name] = value;
};
}
else
{
updateVariableFn = (name, value) =>
{
// deep copy the value
value = PhpVariable.DeepCopy(PhpVariable.Dereference(value));
// set the value to globals
object item;
PhpReference ref_item;
if (globals.TryGetValue(name, out item) && (ref_item = item as PhpReference) != null)
ref_item.Value = value;
else
globals[name] = value;
};
}
}
#endregion
Debug.Assert(updateVariableFn != null);
#region select function that checks if variable exists
if (localVariables != null)
containsFn = (name) => localVariables.ContainsKey(name);
else
containsFn = (name) => globals.ContainsKey(name);
#endregion
Debug.Assert(containsFn != null);
//
//
//
int extracted_count = 0;
foreach (KeyValuePair<IntStringKey, object> entry in vars)
{
string name = entry.Key.ToString();
if (String.IsNullOrEmpty(name) && type != ExtractType.PrefixInvalid) continue;
switch (type)
{
case ExtractType.Overwrite:
// anything is overwritten:
break;
case ExtractType.Skip:
// skips existing name:
if (containsFn(name)) continue;
break;
case ExtractType.IfExists:
// skips nonexistent name:
if (!containsFn(name)) continue;
break;
case ExtractType.PrefixAll:
// prefix anything:
name = String.Concat(prefix, "_", name);
break;
case ExtractType.PrefixInvalid:
// prefixes invalid, others are overwritten:
if (!PhpVariable.IsValidName(name))
name = String.Concat(prefix, "_", name);
break;
case ExtractType.PrefixSame:
// prefixes existing, others are overwritten:
if (containsFn(name))
name = String.Concat(prefix, "_", name);
break;
case ExtractType.PrefixIfExists:
// prefixes existing, others are skipped:
if (containsFn(name))
name = String.Concat(prefix, "_", name);
else
continue;
break;
default:
PhpException.InvalidArgument("type", LibResources.GetString("arg:invalid_value"));
return 0;
}
// invalid names are skipped:
if (PhpVariable.IsValidName(name))
{
// write the value to locals or globals:
updateVariableFn(name, entry.Value);
extracted_count++;
}
}
return extracted_count;
}
public static int Extract(Dictionary<string, object> definedVariables, PhpArray vars, ExtractType type)
{
// GENERICS: IDictionary<string,object>
switch (type & ExtractType.NonFlags)
{
case ExtractType.PrefixSame:
case ExtractType.PrefixAll:
case ExtractType.PrefixInvalid:
case ExtractType.PrefixIfExists:
PhpException.InvalidArgument("prefix", LibResources.GetString("should_be_specified", "prefix"));
return 0;
}
return Extract(definedVariables, vars, type, null);
}
/// <summary>
/// Extract data from the cart
/// </summary>
/// <param name="extractType"></param>
/// <returns>true if successful</returns>
public bool GetData(ExtractType extractType)
{
//choices are update only or full export
//all normal exports include catalog and related files (exclusions, attNames, etc)
//full export also includes all sales files (could be pulling from xml depending on the cart)
//update will include the current sales month only if Rules.ExtractSalesUpdate is true for this cart
if (Rules == null)
{
if (Log != null)
Log.WriteEntry(EventLogEntryType.Information, "Attempt to GetData with Rules null", "", Alias);
return false;
}
IsExtracting = true;
var dynamicUpdate = false;
var totalTime = new StopWatch(true);
var pullType = "new";
//check migration status
_migrationSlave = Rules.MigrationRules != null && Rules.MigrationRules.Enabled
&& !Rules.MigrationRules.IsMigrationMaster;
if (_migrationSlave)
{
pullType = "Migration";
Progress.IsSlave = true;
}
var result = String.Format("Pulling {0} data for {1} from {2}", pullType, Alias, Rules.CartName);
var details = "";
if (Log != null)
Log.WriteEntry(EventLogEntryType.Information, result, "", Alias);
Progress.Start(result);
var catalogCount = 0;
try
{
var generateTables = true;
switch (extractType)
{
case ExtractType.Full:
if (Rules.ExtractSalesFull) GetAllSales();
if (Rules.ExtractCustomerData) GetAllCustomers();
catalogCount = GetCatalog(true);
break;
case ExtractType.Update:
if (Rules.ExtractSalesUpdate) GetSalesUpdate();
if (Rules.ExtractCustomerData) GetCustomerUpdate();
catalogCount = GetCatalog();
break;
case ExtractType.Sales:
if (Rules.ExtractSalesFull || Rules.ExtractSalesFromXmlFile) GetAllSales();
else if (Rules.ExtractSalesUpdate) GetSalesUpdate();
else throw new Exception("Sales extraction is not supported for this site.");
break;
case ExtractType.Customers:
if (Rules.ExtractCustomerData) GetAllCustomers();
else throw new Exception("Customer extraction is not supported for this site.");
break;
case ExtractType.Inventory:
GetInventory();
dynamicUpdate = true;
generateTables = false; //inventory does not require generator
break;
case ExtractType.Catalog:
catalogCount = GetCatalog();
break;
#if DEBUG
case ExtractType.Test:
RunTest();
break;
#endif
case ExtractType.GenerateOnly:
break;
}
//check exclusion causes
var exCauses = "";
if (ExclusionStats != null && ExclusionStats.Any())
{
//only save to rules when extraction is completed
Rules.ExclusionStats = new Dictionary<string,int>(ExclusionStats);
//exclusionStats.AddRange(dashSite.Rules.ExclusionStats.Select(c => new List<string> { c.Key, c.Value.ToString("N0") }));
exCauses = "\n\n" + ExclusionStats
.Aggregate("Exclusion Causes:", (a, b) => String.Format("{0}\n {1} = {2}", a, b.Key, b.Value.ToString("N0")));
//check for excessive missing images
int causeCount;
if (Log != null && ExclusionStats.TryGetValue(MissingImageCause, out causeCount)
&& catalogCount > 0 && ((float)causeCount / catalogCount) * 100F > Rules.MissingImageThreshold) //defaults to 10%
throw new Exception(string.Format("Error: {0} of {1} items are excluded due to missing images.",
causeCount.ToString("N0"), catalogCount.ToString("N0")),
new Exception(exCauses));
}
totalTime.Stop();
if (dynamicUpdate)
{
Rules.LastDynamicUpdateType = extractType;
Rules.LastDynamicUpdateTime = DateTime.Now;
}
else
{
Rules.LastExtractionType = extractType;
Rules.LastExtractionTime = DateTime.Now;
Rules.LastExtractorDuration = (int)Math.Ceiling(totalTime.ElapsedMinutes);
}
if (!_migrationSlave)
{
#if !CART_EXTRACTOR_TEST_SITE
if (generateTables)
{
QueueGenerator();
result = string.Format("Successful update --pending Generator task{0}", exCauses);
}
else
#endif
result = string.Format("Successful {0} update{1}", extractType, exCauses);
}
Progress.End(true, result);
return true;
}
catch (ThreadAbortException)
{
result = "Update canceled by user";
return false;
}
catch (Exception ex)
{
result = ex.Message;
details = ex.InnerException == null ? "" : "Inner Exception: " + ex.InnerException.Message;
if (Log != null)
Log.WriteEntry(EventLogEntryType.Information, result, details, Alias, true); //send support alert
return false;
}
finally
{
if (!_migrationSlave)
{
Rules.QueueSettings(false); //save the extraction time and duration
ReleaseGlobalData();
ReleaseCartData();
}
IsExtracting = false;
if (Progress.Started) Progress.End(false, result);
if (Log != null)
{
Log.WriteEntry(EventLogEntryType.Information, Progress.Text, details, Alias);
}
}
}
private List <string> GetNodeValues(HtmlNodeCollection nodes, List <string> xpathEndAttributes, ExtractType extractType)
{
if (nodes == null || nodes.Count <= 0)
{
return(null);
}
if (xpathEndAttributes != null && xpathEndAttributes.Count > 0)
{
return(nodes.Select(n => n.Attributes.Where(a => xpathEndAttributes.Contains(a.Name)).Select(a => a.Value).FirstOrDefault()).
Where(n => !string.IsNullOrEmpty(n))
.ToList());
}
switch (extractType)
{
case ExtractType.Text:
return(nodes.Select(n => n.InnerText.Trim()).ToList());
case ExtractType.Html:
default:
return(nodes.Select(n => n.InnerHtml.Trim()).ToList());
}
}
public static int Extract(Dictionary <string, object> localVariables, PhpArray /*!*/ vars, ExtractType type,
string prefix)
{
if (vars == null)
{
PhpException.ArgumentNull("vars");
return(0);
}
if (vars.Count == 0)
{
return(0);
}
// unfortunately, type contains flags are combined with enumeration:
bool refs = (type & ExtractType.Refs) != 0;
type &= ExtractType.NonFlags;
//
// construct the action used to set the variable into the locals/globals
//
Action <string /*name*/, object /*value*/> updateVariableFn; // function that writes the value to locals/globals
Predicate <string /*name*/> containsFn; // function that checks if variable exists
PhpArray globals = (localVariables != null) ? null : ScriptContext.CurrentContext.GlobalVariables;
#region select function that writes the variable
if (refs)
{
// makes a reference and writes it back (deep copy is not necessary, "no duplicate pointers" rule preserved):
if (localVariables != null)
{
updateVariableFn = (name, value) =>
{
localVariables[name] = vars[name] = PhpVariable.MakeReference(value);
};
}
else
{
updateVariableFn = (name, value) =>
{
globals[name] = vars[name] = PhpVariable.MakeReference(value);
};
}
}
else
{
if (localVariables != null)
{
updateVariableFn = (name, value) =>
{
// deep copy the value
value = PhpVariable.DeepCopy(PhpVariable.Dereference(value));
// put into locals
object item;
PhpReference ref_item;
if (localVariables.TryGetValue(name, out item) && (ref_item = item as PhpReference) != null)
{
ref_item.Value = value;
}
else
{
localVariables[name] = value;
}
};
}
else
{
updateVariableFn = (name, value) =>
{
// deep copy the value
value = PhpVariable.DeepCopy(PhpVariable.Dereference(value));
// set the value to globals
object item;
PhpReference ref_item;
if (globals.TryGetValue(name, out item) && (ref_item = item as PhpReference) != null)
{
ref_item.Value = value;
}
else
{
globals[name] = value;
}
};
}
}
#endregion
Debug.Assert(updateVariableFn != null);
#region select function that checks if variable exists
if (localVariables != null)
{
containsFn = (name) => localVariables.ContainsKey(name);
}
else
{
containsFn = (name) => globals.ContainsKey(name);
}
#endregion
Debug.Assert(containsFn != null);
//
//
//
int extracted_count = 0;
foreach (KeyValuePair <IntStringKey, object> entry in vars)
{
string name = entry.Key.ToString();
if (String.IsNullOrEmpty(name) && type != ExtractType.PrefixInvalid)
{
continue;
}
switch (type)
{
case ExtractType.Overwrite:
// anything is overwritten:
break;
case ExtractType.Skip:
// skips existing name:
if (containsFn(name))
{
continue;
}
break;
case ExtractType.IfExists:
// skips nonexistent name:
if (!containsFn(name))
{
continue;
}
break;
case ExtractType.PrefixAll:
// prefix anything:
name = String.Concat(prefix, "_", name);
break;
case ExtractType.PrefixInvalid:
// prefixes invalid, others are overwritten:
if (!PhpVariable.IsValidName(name))
{
name = String.Concat(prefix, "_", name);
}
break;
case ExtractType.PrefixSame:
// prefixes existing, others are overwritten:
if (containsFn(name))
{
name = String.Concat(prefix, "_", name);
}
break;
case ExtractType.PrefixIfExists:
// prefixes existing, others are skipped:
if (containsFn(name))
{
name = String.Concat(prefix, "_", name);
}
else
{
continue;
}
break;
default:
PhpException.InvalidArgument("type", LibResources.GetString("arg:invalid_value"));
return(0);
}
// invalid names are skipped:
if (PhpVariable.IsValidName(name))
{
// write the value to locals or globals:
updateVariableFn(name, entry.Value);
extracted_count++;
}
}
return(extracted_count);
}
public static int Extract(Dictionary <string, object> definedVariables, PhpArray vars, ExtractType type)
{
// GENERICS: IDictionary<string,object>
switch (type & ExtractType.NonFlags)
{
case ExtractType.PrefixSame:
case ExtractType.PrefixAll:
case ExtractType.PrefixInvalid:
case ExtractType.PrefixIfExists:
PhpException.InvalidArgument("prefix", LibResources.GetString("should_be_specified", "prefix"));
return(0);
}
return(Extract(definedVariables, vars, type, null));
}
