public IIndex <TKey, TRecord> OpenXIndex <TKey, TRecord>(IDataTransformer <TKey> keyTransformer, IDataTransformer <TRecord> recordTransformer, CompareOption[] compareOptions, bool compressKeys, bool compressRecords, params string[] path)
{
var keyType = keyTransformer.DataType;
bool keySupported = keyType.IsPrimitive || (keyType.IsSlotes && keyType.AreAllTypesPrimitive);
if (!keySupported)
{
throw new NotSupportedException(keyType.ToString());
}
var recordType = recordTransformer.DataType;
bool recordSupported = recordType.IsPrimitive || (recordType.IsSlotes && recordType.AreAllTypesPrimitive);
if (!recordSupported)
{
throw new NotSupportedException(recordType.ToString());
}
Type[] keySlotTypes = keyType.IsPrimitive ? new Type[] { keyType.PrimitiveType } : keyType.Select(x => x.PrimitiveType).ToArray();
Type[] recordSlotTypes = recordType.IsPrimitive ? new Type[] { recordType.PrimitiveType } : recordType.Select(x => x.PrimitiveType).ToArray();
keyType.CheckCompareOptions(compareOptions);
KeyDescriptor keyDescriptor = new KeyDescriptor(keyType, compressKeys, compareOptions);
RecordDescriptor recordDescriptor = new RecordDescriptor(recordType, compressRecords);
var index = OpenXIndex(keyDescriptor, recordDescriptor, path);
return(new XIndex <TKey, TRecord>(index, keyTransformer, recordTransformer));
}
public FunctionsCalculator(IEnumerable <TrainingElement> trainingSet)
{
this.TrainingSet = trainingSet.ToList();
this.valuesNormalizer = new ValueNormalizer();
this.dataSupplier = new DataSourceSupplier();
this.dataTransformer = new DataTransformer();
}
public IEnumerable ReadAll(IDataTransformer transformer)
{
IList results = new ArrayList();
var reader = CreateReader();
try
{
reader.Open();
string[] aliases = null;
if (_dialect.HasHeader)
{
aliases = reader.Headers;
if (aliases == null)
{
throw new HeaderIsNullException("Failed to read headers");
}
}
while (reader.Next())
{
results.Add(transformer.TransformTuple(reader.Current, aliases));
}
}
finally
{
reader.Dispose();
}
return(transformer.TransformResult(results));
}
public IEnumerable ReadAll(IDataTransformer transformer)
{
IList results = new ArrayList();
var reader = CreateReader();
var row = 0;
string[] aliases = null;
if (_dialect.HasHeader)
{
aliases = reader.Headers;
if (aliases == null)
{
throw new HeaderIsNullException("Failed to read headers");
}
}
while (reader.Next())
{
var values = reader.Current;
results.Add(transformer.TransformTuple(values, aliases));
row++;
}
reader.Dispose();
return transformer.TransformResult(results);
}
public Executor(IUserInterface userInterface, IFileLoader fileLoader, IDataTransformer dataTransformer, IPrecipitationRepository precipitationRepository)
{
_fileLoader = fileLoader;
_userInterface = userInterface;
_dataTransformer = dataTransformer;
_precipitationRepository = precipitationRepository;
}
/// <summary>
/// Clones the current stored procedure, and adds the <paramref name="transformer"/> that will attempt
/// to transform every value in the result set(s) (for every column).
/// </summary>
/// <param name="transformer">The <see cref="IDataTransformer"/> to use to transform values in the results.</param>
/// <returns>A copy of the current <see cref="StoredProcedure"/> with the additional transformer.</returns>
protected internal StoredProcedure CloneWith(IDataTransformer transformer)
{
Contract.Requires(transformer != null);
Contract.Ensures(Contract.Result <StoredProcedure>() != null);
return(CloneCore(parameters, dataTransformers.Concat(new[] { transformer })));
}
public LocalStorageService(IJSRuntime jSRuntime, IOptions <LocalStorageOptions> options, IDataTransformer dataTransformer)
{
_jSRuntime = jSRuntime;
_jsonOptions = options.Value.JsonSerializerOptions;
_dataTransformer = dataTransformer;
_jSInProcessRuntime = jSRuntime as IJSInProcessRuntime;
}
public Core(IInPortBlock inPorts, IOutPortBlock outPorts, IRouter router, IDataTransformer dataTransformer)
{
InPorts = inPorts;
OutPorts = outPorts;
Router = router;
DataTransformer = dataTransformer;
WaitedPackets = new HashSet <IPacket>();
}
protected static bool IsTypedTransformer(IDataTransformer transformer)
{
Contract.Requires(transformer != null);
return(transformer.GetType()
.GetInterfaces()
.Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IDataTransformer <>)));
}
public FunctionsCalculator(
IValuesNormalizer valuesNormalizer,
IDataSourceSupplier dataSupplier,
IDataTransformer dataTransformer)
{
this.valuesNormalizer = valuesNormalizer;
this.dataSupplier = dataSupplier;
this.dataTransformer = dataTransformer;
}
public FlowTransformer(IDataTransformer <T> transformer)
{
Transfromer = transformer;
foreach (var item in new System.Management.ManagementObjectSearcher("Select * from Win32_Processor").Get())
{
coreCount += int.Parse(item["NumberOfCores"].ToString());
}
}
/// <summary>
/// Clones the given <see cref="StoredProcedure"/> with a new <see cref="IDataTransformer"/> that
/// will be used to transform data returned from the database.
/// </summary>
/// <typeparam name="TSP">The <see cref="StoredProcedure"/> type that this is applied to. This is needed so the fluent API works correctly.</typeparam>
/// <param name="sp">The <see cref="StoredProcedure"/> to clone and associated the transformer with.</param>
/// <param name="transformer">The <see cref="IDataTransformer"/> to use to transform all data returned by the database.</param>
/// <returns>A copy of the <see cref="StoredProcedure"/> with the <see cref="IDataTransformer"/> associated.</returns>
/// <remarks>Every column returned will attempt to transform data through this transformer, unless all transformers used implement
/// <see cref="IDataTransformer{T}"/>, in which case only the generic <see cref="IDataTransformer{T}.Transform"/> will be called
/// for columns that are the matching type for the transformer.</remarks>
/// <example>
/// <code language="cs">
/// var sp = StoredProcedure.Create("usp_GetPeople")
/// .WithResults<Person>()
/// .WithDataTransformer(new InternAllStringsTransformer());
/// </code>
/// </example>
public static TSP WithDataTransformer <TSP>(this TSP sp, IDataTransformer transformer)
where TSP : StoredProcedure
{
Contract.Requires(sp != null);
Contract.Requires(transformer != null);
Contract.Ensures(Contract.Result <TSP>() != null);
return((TSP)sp.CloneWith(transformer));
}
public List<FileEntry> GetDirectory(Stream strm, IDataTransformer zlibdec)
{
List<FileEntry> results = new List<FileEntry>();
byte[] headerbytes = new byte[512];
int i = 0;
byte b;
string[] header;
int IndexOffset;
int Key;
byte[] IndexData;
byte[] OutIndexData;
int OutLength = 0;
// read header
do
{
b = (byte)strm.ReadByte();
headerbytes[i++] = b;
} while (b != 0x0a);
header = Encoding.ASCII.GetString(headerbytes, 0, i - 1).Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (header.Length != 3)
return null;
IndexOffset = Convert.ToInt32(header[1], 16);
Key = Convert.ToInt32(header[2], 16);
// read index data
strm.Seek(IndexOffset, SeekOrigin.Begin);
IndexData = new byte[strm.Length - IndexOffset];
strm.Read(IndexData, 0, IndexData.Length);
OutIndexData = new byte[2 * IndexData.Length];
do
{
TransformationResult res;
try
{
OutLength = OutIndexData.Length;
res = zlibdec.TransformData(IndexData, OutIndexData, IndexData.Length, ref OutLength);
}
catch (Exception)
{
res = TransformationResult.BufferTooSmall;
}
if (res != TransformationResult.BufferTooSmall)
break;
// if our buffer is too small, try again with a bigger one
OutIndexData = new byte[2 * OutIndexData.Length];
} while (true);
results = PickleLoader.Load(OutIndexData, Key);
return results;
}
public Builder AddVolumeSum()
{
if (_config.VolumeSum)
{
_hasAggFunc = true;
_service = new VolumeSumReportTransformer(_service);
}
return(this);
}
public Builder AddWeightSum()
{
if (_config.WeightSum)
{
_hasAggFunc = true;
_service = new WeightSumReportTransfomer(_service);
}
return(this);
}
public Builder AddCostSum()
{
if (_config.WithData)
{
_hasAggFunc = true;
_service = new WithDataReportTransformer(_service);
}
return(this);
}
public void Add(IDataTransformer dataTransformer, bool isPreProcessor)
{
if (isPreProcessor)
{
PreProcessors.Add(dataTransformer);
}
else
{
PostProcessors.Add(dataTransformer);
}
}
/// <summary>
/// Ctor
/// </summary>
/// <param name="extractor">Data extractor</param>
/// <param name="cleanser">Data cleanser</param>
/// <param name="transformer">Data transformation</param>
/// <param name="loader">Data loader</param>
public ETLProcessor(
IDataExtractor <T> extractor,
IDataCleanser <T> cleanser,
IDataTransformer <T, U> transformer,
IDataLoader <U> loader)
{
_extractor = extractor;
_cleanser = cleanser;
_transformer = transformer;
_loader = loader;
}
public TradesExtrator(
ITaxReport taxReport,
IBrokerParserProvider brokerParserProvider,
IDataTransformer <Trade, ClosedOperation <Trade> > tradeTransformer,
IEntityConvertor <Trade> tradeConvertor
)
{
this.taxReport = taxReport;
this.brokerParserProvider = brokerParserProvider;
this.tradeConvertor = tradeConvertor;
this.tradeTransformer = tradeTransformer;
}
public IDataTransformer Get(string type)
{
IDataTransformer transformer = basetable.Get(type);
if (transformer != null)
{
return(transformer);
}
instancetransformers.TryGetValue(type, out transformer);
return(transformer);
}
/// <summary>
/// Ctor
/// </summary>
/// <param name="tokenizer"></param>
/// <param name="transformer"></param>
public TransformTokenizer(ITextTokenizer <TIn> tokenizer, IDataTransformer <TIn, TOut> transformer)
{
if (tokenizer == null)
{
new ArgumentNullException("tokenizer");
}
if (transformer == null)
{
new ArgumentNullException("transformer");
}
this.Tokenizer = tokenizer;
this.Transformer = transformer;
}
public IIndex <TKey, TRecord> OpenXIndex <TKey, TRecord>(IDataTransformer <TKey> keyTransformer, IDataTransformer <TRecord> recordTransformer, CompareOption[] compareOptions, bool compressKeys, bool compressRecords, params string[] path)
{
DataType keyType = keyTransformer.DataType;
DataType recordType = recordTransformer.DataType;
keyType.CheckCompareOptions(compareOptions);
KeyDescriptor keyDescriptor = new KeyDescriptor(keyType, compressKeys, compareOptions);
RecordDescriptor recordDescriptor = new RecordDescriptor(recordType, compressRecords);
var index = OpenXIndex(keyDescriptor, recordDescriptor, path);
return(new XIndex <TKey, TRecord>(index, keyTransformer, recordTransformer));
}
/// <summary>
/// Ctor
/// </summary>
/// <param name="formatter"></param>
/// <param name="transformer"></param>
public TransformFormatter(ITextFormatter <TOut> formatter, IDataTransformer <TIn, TOut> transformer)
{
if (formatter == null)
{
new ArgumentNullException("formatter");
}
if (transformer == null)
{
new ArgumentNullException("transformer");
}
this.Formatter = formatter;
this.Transformer = transformer;
}
/// <summary>
///
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="transformer"></param>
/// <param name="sheetNumber"></param>
/// <param name="intRemoveRows">Số row cần bỏ qua (không tính dòng đầu tiên)</param>
/// <returns></returns>
public List <T> GetData <T>(IDataTransformer transformer, int sheetNumber = 0, int intRemoveRows = 0)
{
var dt = data.Tables[sheetNumber];
DataTable tb = dt as DataTable;
if (intRemoveRows > 0 && tb != null && tb.Rows.Count > intRemoveRows)
{
for (int i = 0; i < intRemoveRows; i++)
{
tb.Rows.RemoveAt(0);
}
}
return(transformer.GetList <T>(tb));
}
public IEnumerable ReadAll(IDataTransformer transformer)
{
IList results = new ArrayList();
CSVReader reader = CreateReader();
int row = 0;
string[] aliases = null;
while (reader.NextRecord())
{
string[] values = reader.GetCurrentRecord();
if (row == 0 && _dialect.HasHeader)
{
aliases = values;
}
else if (_types.Count > 0)
{
object[] record = new object[values.Length];
int count = values.Length;
for (int i = 0; i < count; i++)
{
if (_types.ContainsKey(aliases[i]))
{
record[i] = ConvertTo(_types[aliases[i]], values[i]);
}
else
{
record[i] = values[i];
}
}
results.Add(transformer.TransformTuple(record, aliases));
}
else
{
results.Add(transformer.TransformTuple(values, aliases));
}
row++;
}
reader.Dispose();
reader = null;
return transformer.TransformResult(results);
}
public void WriteAll(IEnumerable data, IDataTransformer transformer)
{
if (transformer == null)
{
throw new DataTransformerIsNullException();
}
var writer = CreateWriter();
try
{
writer.Open();
var cellCount = -1;
if (_dialect.HasHeader && _headers != null && _headers.Length > 0)
{
writer.WriteRow(_headers);
}
foreach (var row in data)
{
var cells = transformer.TransformRow(row);
if (cells != null)
{
if (cellCount == -1)
{
cellCount = cells.Length;
}
if (cells.Length != cellCount)
{
throw new NotEqualCellCountInRowsException(cells.Length, cellCount);
}
}
writer.WriteRow(cells);
}
}
finally
{
writer.Dispose();
}
}
IJavaData ReduceObject(JavaObject @object, HashSet <JavaObject> visited)
{
if (visited.Contains(@object))
{
return(@object);
}
visited.Add(@object);
foreach (JavaField field in @object.Fields)
{
if (field.Value is JavaArray)
{
ReduceArray((JavaArray)field.Value, visited);
}
else if (field.Value is JavaObject)
{
field.Value = ReduceObject((JavaObject)field.Value, visited);
}
}
for (int i = @object.Custom.Count - 1; i >= 0; --i)
{
IJavaData data = @object.Custom[i];
if (data is JavaArray)
{
ReduceArray((JavaArray)data, visited);
}
else if (data is JavaObject)
{
@object.Custom[i] = ReduceObject((JavaObject)data, visited);
}
}
visited.Remove(@object);
IDataTransformer transformer = transformationtable.Get(@object.Type);
if (transformer == null)
{
return(@object);
}
return(transformer.Convert(@object));
}
/// <summary>
/// Executes a transform
/// </summary>
/// <param name="transform">The transform to run</param>
/// <param name="aValue">The A value to pass to the transform</param>
/// <param name="bValue">The B value to pass to the transform</param>
/// <returns>The result of the transform to be assigned back to the scope (or to the values in the case of unary transforms)</returns>
internal static object ExecuteTransform(String transformName, object[] transformParameters, ref object aValue, ref object bValue)
{
try
{
IDataTransformer transformer = null;
if (s_transforms.TryGetValue(transformName, out transformer))
{
if (transformer is IUnaryDataTransformer)
{
#if DEBUG
m_tracer.TraceVerbose("Will apply unary transform {0} on {1}", transformName, aValue);
#endif
aValue = (transformer as IUnaryDataTransformer).Apply(aValue, transformParameters);
#if DEBUG
m_tracer.TraceVerbose("Will apply unary transform {0} on {1}", transformName, bValue);
#endif
bValue = (transformer as IUnaryDataTransformer).Apply(bValue, transformParameters);
return(null); // No result for unary transforms
}
else if (transformer is IBinaryDataTransformer)
{
#if DEBUG
m_tracer.TraceVerbose("Will apply binary transform {0} on {1} and {2}", transformName, aValue, bValue);
#endif
return((transformer as IBinaryDataTransformer).Apply(aValue, bValue, transformParameters));
}
else
{
throw new InvalidOperationException("Invalid type of transform");
}
}
else
{
throw new KeyNotFoundException($"Transform {transformName} is not registered");
}
}
catch (Exception e)
{
m_tracer.TraceError("Error executing {0} on {1} and {2} - {3}", transformName, aValue, bValue, e.Message);
throw new MatchingException($"Error executing {transformName} on {aValue} and {bValue}", e);
}
}
public ReportServiceTransformerBase GetServiceTransformer(string arg, IDataTransformer service)
{
switch (arg)
{
case ArgsConst.Data:
return(new WithDataReportTransformer(service));
case ArgsConst.VolumeSum:
return(new VolumeSumReportTransformer(service));
case ArgsConst.WeightSum:
return(new WeightSumReportTransfomer(service));
case ArgsConst.CostSum:
return(new CostSumReportTransformer(service));
case ArgsConst.CountSum:
return(new CountSumReportTransformer(service));
default:
throw new NotSupportedException("This arg not supported");
}
}
public CostSumReportTransformer(IDataTransformer dataTransformer)
: base(dataTransformer)
{
}
public WithDataReportTransformer(IDataTransformer reportService) : base(reportService)
{
}
protected DataTransformerBase(IDataTransformer dataTransformer)
{
DataTransformer = dataTransformer;
}
protected ReportServiceTransformerBase(IDataTransformer dataTransformer)
{
DataTransformer = dataTransformer;
}
public void AddTransformer(IDataTransformer trn)
{
m_transformers.Add(trn.GetName(), trn.GetType());
}
public XORStream(Stream basestream, IDataTransformerRegistry reg, byte value)
{
_baseStream = basestream;
_transformer = reg.GetTransformer("xor");
_transformer.SetOption("value", value);
}
public void WriteAll(IEnumerable data, IDataTransformer transformer)
{
bool isFirst = true;
CSVWriter writer = CreateWriter();
foreach (object row in data)
{
if (isFirst && _dialect.HasHeader)
{
string[] stringRow = row as string[];
if (stringRow != null)
{
writer.WriteRow(stringRow);
stringRow = null;
}
isFirst = false;
}
else
{
writer.WriteRow(transformer.TransformRow(row));
}
}
writer.Dispose();
writer = null;
}
public void WriteAll(IEnumerable data, IDataTransformer transformer)
{
if (transformer == null)
{
throw new DataTransformerIsNullException();
}
var cellCount = -1;
var writer = CreateWriter();
if (_dialect.HasHeader && _headers != null && _headers.Length > 0)
{
writer.WriteRow(_headers);
}
foreach (var row in data)
{
var cells = transformer.TransformRow(row);
if (cells != null)
{
if (cellCount == -1)
{
cellCount = cells.Length;
}
if (cells.Length != cellCount)
{
throw new NotEqualCellCountInRowsException(cells.Length, cellCount);
}
}
writer.WriteRow(cells);
}
writer.Dispose();
}
