public bool DetectFromDataTable(ExtractionType et, DataTable dt)
{
return(dt.Columns.Cast <DataColumn>().Any(
c => (c.ColumnName == "query_hash" && et == ExtractionType.QueryPlans)
||
((c.ColumnName == "query_hash" || c.ColumnName == "sql_hash") && et == ExtractionType.SqlText)));
}
private void Extract(ExtractionType et)
{
FormConnectToSqlServer formConnectToSqlServer = new FormConnectToSqlServer();
formConnectToSqlServer.ShowDialog();
ExtractionService eService = new ExtractionService();
switch (et)
{
case ExtractionType.ExtractAll:
eService.ExtractAll(this.textBoxSource.Text, false);
break;
case ExtractionType.ExtractFunctions:
eService.ExtractFunctions(this.textBoxSource.Text, false);
break;
case ExtractionType.ExtractStoredProcedures:
eService.ExtractStoredProcedures(this.textBoxSource.Text, false);
break;
case ExtractionType.ExtractViews:
break;
default:
eService.ExtractAll(this.textBoxSource.Text, false);
break;
}
this.listBoxSourceFiles.DataSource = Directory.GetFiles(textBoxSource.Text);
}
public bool DetectFromDataTable(ExtractionType et, DataTable dt)
{
return dt.Columns.Cast<DataColumn>().Any(
c => (c.ColumnName == "query_hash" && et == ExtractionType.QueryPlans)
||
((c.ColumnName == "query_hash" || c.ColumnName == "sql_hash") && et == ExtractionType.SqlText));
}
public string GetAccountExtractionConfiguration(HttpListenerContext ct, ActionInfo hi)
{
ExtractionConfiguration Ec = new ExtractionConfiguration();
Ec.DefaultDocumentSpecification = "Stuff";
Ec.OrientationDetectionMode = OrientationDetectionMode.Off;
DocumentType dt = new DocumentType();
dt.Country = "SE";
dt.DisplayName = "PDF";
dt.ExtractionName = "Thefil.pdf";
FieldGroup Fg = new FieldGroup();
Fg.IsTableGroup = true;
Fg.Name = "My Group";
Fg.Order = 23;
dt.FieldGroups.Add(Fg);
DocumentTypeField dtf = new DocumentTypeField();
dtf.FormatType = FormatType.RegularExpression;
ExtractionType et = new ExtractionType();
dtf.ExtractionType = et;
dt.Fields.Add(dtf);
dt.InternalName = "hejsan";
dt.SystemName = "Thesystemname";
dt.IsSelected = true;
Ec.SelectedDocumentTypes.Add(dt);
return(GetResponseString <ExtractionConfiguration>(Ec, ct));
}
public bool ExtractFromDataTable(ExtractionType et, DataTable dt)
{
if (!DetectFromDataTable(et, dt))
{
return(false);
}
var col = (dt.Columns.Cast <DataColumn>().First(
c => (c.ColumnName == "query_hash" && et == ExtractionType.QueryPlans)
||
((c.ColumnName == "query_hash" || c.ColumnName == "sql_hash") && et == ExtractionType.SqlText))).Ordinal;
HandlesList = dt.Rows.Cast <DataRow>().Select(r => r.Field <String>(col)).ToDictionary(k => k, v => new CacheObject());
if (Extract(et))
{
try
{
SaveAll(et);
}
catch (Exception e)
{
SimpleLog.Error(e.Message);
return(false);
}
}
else
{
return(false);
}
return(true);
}
public bool Extract(ExtractionType et)
{
var qry = new StringBuilder("DECLARE @params as XML;");
qry.Append($"SET @params = '{XmlParameterList()}';");
qry.Append("WITH params AS (SELECT p.value('.', 'varbinary(64)') as handle FROM @params.nodes('/params/p') n(p)) "); // TODO
switch (et)
{
case ExtractionType.QueryPlans:
qry.Append("SELECT params.handle, qp.query_plan FROM params CROSS APPLY sys.dm_exec_query_plan(params.handle) as qp");
break;
case ExtractionType.SqlText:
qry.Append("SELECT params.handle, st.text FROM params CROSS APPLY sys.dm_exec_sql_text(params.handle) as st");
break;
default:
//throw new InvalidOperationException("Unknown ExtractionType");
return false;
break;
}
try
{
if (_cn.State == ConnectionState.Closed) _cn.Open();
using (var cmd = new SqlCommand(qry.ToString(), _cn))
{
var rd = cmd.ExecuteReader();
while (rd.Read())
{
var co = new CacheObject();
switch (et)
{
case ExtractionType.QueryPlans:
co.PlanHandle = rd.GetString(0);
co.QueryPlan = rd.GetString(1);
break;
case ExtractionType.SqlText:
co.SqlHandle = rd.GetString(0);
co.SqlText = rd.GetString(1);
break;
default:
break;
}
HandlesList[rd.GetString(0)] = co;
}
rd.Close();
}
}
catch (SqlException e)
{
var msg =
$"SQL exception : {e.Number}, {e.Message}";
SimpleLog.Error(msg);
return false;
}
return true;
}
public async Task RunAsync(ExtractionType extractionType, int startBlock, bool distribute = false)
{
if (extractionType == ExtractionType.Swap)
{
Console.WriteLine($"Scanning for swap transactions...");
for (int height = startBlock; height < EndHeight; height++)
{
BlockTransactionDetailsModel block = await RetrieveBlockAtHeightAsync(height);
ProcessBlockForSwapTransactions(block, height);
}
Console.WriteLine($"Found {this.swapTransactions.Count} swap transactions.");
using (var writer = new StreamWriter(Path.Combine("c:", "[StratisWork]", "swaps.csv")))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
foreach (SwapTransaction swapTransaction in this.swapTransactions)
{
csv.WriteRecord(swapTransaction);
}
}
if (distribute)
{
await BuildAndSendDistributionTransactionsAsync();
}
}
if (extractionType == ExtractionType.Vote)
{
Console.WriteLine($"Scanning for votes...");
for (int height = startBlock; height < EndHeight; height++)
{
BlockTransactionDetailsModel block = await RetrieveBlockAtHeightAsync(height);
if (block == null)
{
break;
}
await ProcessBlockForVoteTransactionsAsync(block, height);
}
IEnumerable <IGrouping <string, CastVote> > grouped = this.castVotes.GroupBy(a => a.Address);
var finalVotes = new List <CastVote>();
foreach (IGrouping <string, CastVote> group in grouped)
{
IOrderedEnumerable <CastVote> finalVote = group.OrderByDescending(t => t.BlockHeight);
finalVotes.Add(finalVote.First());
}
Console.WriteLine($"Total No Votes: {finalVotes.Count(v => !v.InFavour)} [Weight : {Money.Satoshis(finalVotes.Where(v => !v.InFavour).Sum(v => v.Balance)).ToUnit(MoneyUnit.BTC)}]");
Console.WriteLine($"Total Yes Votes: {finalVotes.Count(v => v.InFavour)} [Weight : {Money.Satoshis(finalVotes.Where(v => v.InFavour).Sum(v => v.Balance)).ToUnit(MoneyUnit.BTC)}]");
}
}
private static string GetContent(ClaimLocation location, ExtractionType extType, HttpRequest req, RouteValueDictionary routeValues, string requestBodyString = "")
{
switch (location)
{
case ClaimLocation.Body:
{
if (!string.IsNullOrEmpty(requestBodyString))
{
return(requestBodyString);
}
var bodyMemStream = new MemoryStream();
if (req.Body.CanSeek)
{
req.Body.Position = 0;
}
req.Body.CopyTo(bodyMemStream);
var stringBody = System.Text.Encoding.UTF8.GetString(bodyMemStream.ToArray());
return(stringBody);
}
case ClaimLocation.Headers:
{
return(extType == ExtractionType.KeyValue ? JsonConvert.SerializeObject(HeaderDictToKvp(req.Headers, req.Headers)) : string.Empty);
}
case ClaimLocation.Uri:
{
if (extType == ExtractionType.KeyValue)
{
return(routeValues == null ? "" : JsonConvert.SerializeObject(RouteValueDictionaryToKvp(routeValues)));
}
if (extType == ExtractionType.RegEx)
{
return(req.Path);
}
break;
}
case ClaimLocation.QueryParameters:
{
switch (extType)
{
case ExtractionType.KeyValue:
return(JsonConvert.SerializeObject(QueryCollectionToKvp(req.Query)));
case ExtractionType.RegEx:
return(req.QueryString.ToString());
}
break;
}
default:
return(string.Empty);
}
return(string.Empty);
}
protected ExtractionRule(ExtractionType type, string path, string property)
{
if (!Enum.IsDefined(typeof(ExtractionType), type))
{
throw new InvalidEnumArgumentException(nameof(type), (int)type, typeof(ExtractionType));
}
Type = type;
Path = path ?? throw new ArgumentNullException(nameof(path));
Property = property ?? throw new ArgumentNullException(nameof(property));
}
/// <summary>
/// Extract fingerprint featuries.
/// </summary>
public byte[] Extract(FingerprintImage fingerprintImage, ExtractionType extractionType)
{
if (!isInitialized)
{
throw new InvalidOperationException("Initialize FingerprintFeatureExtractor first.");
}
if (isDisposed)
{
throw new ObjectDisposedException(GetType().Name);
}
if (fingerprintImage == null)
{
throw new ArgumentNullException("fingerprintImage");
}
var featureSetSizeRecommended = 0;
var featureSetSizeMinimum = 0;
CalculateFeatureSizeCore(extractionType, ref featureSetSizeRecommended, ref featureSetSizeMinimum);
if (featureSetSizeRecommended <= 0)
{
throw new OutOfMemoryException();
}
var fdFeatures = Marshal.AllocHGlobal(featureSetSizeRecommended);
var pImageQuality = (int)FT_IMG_QUALITY.FT_UNKNOWN_IMG_QUALITY;
var pFeaturesQualit = (int)FT_FTR_QUALITY.FT_UNKNOWN_FTR_QUALITY;
var featureSetCreated = 0;
ExtractFeaturesCore(fxContextHandle, fingerprintImage.OriginalFingerprintImageSize, fingerprintImage.OriginalFingerprintImage, (int)extractionType,
featureSetSizeRecommended, fdFeatures, ref pImageQuality, ref pFeaturesQualit, ref featureSetCreated);
if (featureSetCreated == 0 ||
pImageQuality != (int)FT_IMG_QUALITY.FT_GOOD_IMG ||
pFeaturesQualit != (int)FT_FTR_QUALITY.FT_GOOD_FTR)
{
throw new InvalidOperationException("Cannot extract features.");
}
var featuriesData = new byte[featureSetSizeRecommended];
Marshal.Copy(fdFeatures, featuriesData, 0, featureSetSizeRecommended);
Marshal.FreeHGlobal(fdFeatures);
return(featuriesData);
}
public void SaveAll(ExtractionType et)
{
var outputFolder = $"{OutputParentFolder}\\{et.ToString()}\\";
string extension;
switch (et)
{
case ExtractionType.QueryPlans:
extension = ".sqlplan";
break;
case ExtractionType.SqlText:
extension = ".sql";
break;
default:
throw new ArgumentOutOfRangeException(nameof(et), et, null);
}
foreach (var co in HandlesList)
{
var file = $"{outputFolder}{co.Key}{extension}";
switch (et)
{
case ExtractionType.QueryPlans:
File.WriteAllText(file, co.Value.QueryPlan);
break;
case ExtractionType.SqlText:
File.WriteAllText(file, co.Value.SqlText);
break;
default:
throw new ArgumentOutOfRangeException(nameof(et), et, null);
}
co.Value.OutputFile = file;
}
}
/// <summary>
/// Get list of positions of the specified word
/// </summary>
/// <param name="sentence"></param>
/// <param name="word"></param>
/// <returns>List of positions</returns>
public static List <int> GetWordPositions(this string sentence, string word, ExtractionType extractionType = ExtractionType.All)
{
int pos = 0;
List <int> lstCount = new List <int>();
bool enableBreak = false;
if (extractionType == ExtractionType.FirstOccurance)
{
enableBreak = true;
}
while ((pos < sentence.Length) && (pos = sentence.IndexOf(word, pos)) > -1)
{
lstCount.Add(pos);
pos += word.Length;
if (enableBreak)
{
break;
}
}
return(lstCount);
}
static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List <string> extractedFiles, Action <string> updateMessage)
{
// Yaml path may be specified relative to a named directory (e.g. ^SupportDir) or the detected disc path
var sourcePath = actionYaml.Value.StartsWith("^") ? Platform.ResolvePath(actionYaml.Value) : Path.Combine(path, actionYaml.Value);
using (var source = File.OpenRead(sourcePath))
{
foreach (var node in actionYaml.Nodes)
{
var targetPath = Platform.ResolvePath(node.Key);
if (File.Exists(targetPath))
{
Log.Write("install", "Skipping installed file " + targetPath);
continue;
}
var offsetNode = node.Value.Nodes.FirstOrDefault(n => n.Key == "Offset");
if (offsetNode == null)
{
Log.Write("install", "Skipping entry with missing Offset definition " + targetPath);
continue;
}
var lengthNode = node.Value.Nodes.FirstOrDefault(n => n.Key == "Length");
if (lengthNode == null)
{
Log.Write("install", "Skipping entry with missing Length definition " + targetPath);
continue;
}
var length = FieldLoader.GetValue <int>("Length", lengthNode.Value.Value);
source.Position = FieldLoader.GetValue <int>("Offset", offsetNode.Value.Value);
extractedFiles.Add(targetPath);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
Action <long> onProgress = null;
if (length < ShowPercentageThreshold)
{
updateMessage("Extracting " + displayFilename);
}
else
{
onProgress = b => updateMessage($"Extracting {displayFilename} ({100 * b / length}%)");
}
using (var target = File.OpenWrite(targetPath))
{
Log.Write("install", $"Extracting {sourcePath} -> {targetPath}");
if (type == ExtractionType.Blast)
{
Blast.Decompress(source, target, (read, _) => onProgress?.Invoke(read));
}
else
{
CopyStream(source, target, length, onProgress);
}
}
}
}
}
/// <summary>
/// method to evaluate a regular expression and return the first match, last match or all matches in a concatenated string
/// </summary>
/// <param name="parseString">string to parse</param>
/// <param name="matchPattern">regular expression to evaluate</param>
/// <param name="extractType">match(es) to return - first|last|all</param>
/// <param name="separator">string to use as a separator for an "all" match return type</param>
/// <returns></returns>
private string ExtractMatchingString(string parseString, string matchPattern, ExtractionType extractType, string separator)
{
//set the default output to the empty string. if we match something, we'll change it.
string output = string.Empty;
//do the regex match
MatchCollection matches = Regex.Matches(parseString, matchPattern);
if (matches.Count > 0)
{
//which match(es) should we return?
switch (extractType)
{
case ExtractionType.First:
output = matches[0].Value;
break;
case ExtractionType.Last:
output = matches[matches.Count - 1].Value;
break;
case ExtractionType.All:
StringBuilder matchingSb = new StringBuilder();
for (int i = 0; i < matches.Count; i++)
{
matchingSb.Append(matches[i].Value);
if (i != matches.Count - 1)
{
matchingSb.Append(separator);
}
}
output = matchingSb.ToString();
break;
}
}
return output;
}
/// <summary>
/// method to evaluate a regular expression and return the first match, last match or all matches in a concatenated string
/// </summary>
/// <param name="parseString">string to parse</param>
/// <param name="matchPattern">regular expression to evaluate</param>
/// <param name="extractType">match(es) to return - first|last|all</param>
/// <param name="separator">string to use as a separator for an "all" match return type</param>
/// <returns></returns>
private string ExtractMatchingString(string parseString, string matchPattern, ExtractionType extractType, string separator)
{
//set the default output to the empty string. if we match something, we'll change it.
string output = string.Empty;
//do the regex match
MatchCollection matches = Regex.Matches(parseString, matchPattern);
if (matches.Count > 0)
{
//which match(es) should we return?
switch (extractType)
{
case ExtractionType.First:
output = matches[0].Value;
break;
case ExtractionType.Last:
output = matches[matches.Count - 1].Value;
break;
case ExtractionType.All:
StringBuilder matchingSb = new StringBuilder();
for (int i = 0; i < matches.Count; i++)
{
matchingSb.Append(matches[i].Value);
if (i != matches.Count - 1)
{
matchingSb.Append(separator);
}
}
output = matchingSb.ToString();
break;
}
}
return(output);
}
public bool ExtractFromDataTable(ExtractionType et, DataTable dt)
{
if (!DetectFromDataTable(et, dt)) return false;
var col = (dt.Columns.Cast<DataColumn>().First(
c => (c.ColumnName == "query_hash" && et == ExtractionType.QueryPlans)
||
((c.ColumnName == "query_hash" || c.ColumnName == "sql_hash") && et == ExtractionType.SqlText))).Ordinal;
HandlesList = dt.Rows.Cast<DataRow>().Select(r => r.Field<String>(col)).ToDictionary(k => k, v => new CacheObject() );
if (Extract(et))
{
try
{
SaveAll(et);
}
catch (Exception e)
{
SimpleLog.Error(e.Message);
return false;
}
}
else
{
return false;
}
return true;
}
public void SaveAll(ExtractionType et)
{
var outputFolder = $"{OutputParentFolder}\\{et.ToString()}\\";
string extension;
switch (et)
{
case ExtractionType.QueryPlans:
extension = ".sqlplan";
break;
case ExtractionType.SqlText:
extension = ".sql";
break;
default:
throw new ArgumentOutOfRangeException(nameof(et), et, null);
}
foreach (var co in HandlesList)
{
var file = $"{outputFolder}{co.Key}{extension}";
switch (et)
{
case ExtractionType.QueryPlans:
File.WriteAllText(file, co.Value.QueryPlan);
break;
case ExtractionType.SqlText:
File.WriteAllText(file, co.Value.SqlText);
break;
default:
throw new ArgumentOutOfRangeException(nameof(et), et, null);
}
co.Value.OutputFile = file;
}
}
public DocumentBuilder WithExtractionType(ExtractionType extractionType)
{
this.extractionTypes.Add(extractionType.ToString());
return(this);
}
public bool Extract(ExtractionType et)
{
var qry = new StringBuilder("DECLARE @params as XML;");
qry.Append($"SET @params = '{XmlParameterList()}';");
qry.Append("WITH params AS (SELECT p.value('.', 'varbinary(64)') as handle FROM @params.nodes('/params/p') n(p)) "); // TODO
switch (et)
{
case ExtractionType.QueryPlans:
qry.Append("SELECT params.handle, qp.query_plan FROM params CROSS APPLY sys.dm_exec_query_plan(params.handle) as qp");
break;
case ExtractionType.SqlText:
qry.Append("SELECT params.handle, st.text FROM params CROSS APPLY sys.dm_exec_sql_text(params.handle) as st");
break;
default:
//throw new InvalidOperationException("Unknown ExtractionType");
return(false);
break;
}
try
{
if (_cn.State == ConnectionState.Closed)
{
_cn.Open();
}
using (var cmd = new SqlCommand(qry.ToString(), _cn))
{
var rd = cmd.ExecuteReader();
while (rd.Read())
{
var co = new CacheObject();
switch (et)
{
case ExtractionType.QueryPlans:
co.PlanHandle = rd.GetString(0);
co.QueryPlan = rd.GetString(1);
break;
case ExtractionType.SqlText:
co.SqlHandle = rd.GetString(0);
co.SqlText = rd.GetString(1);
break;
default:
break;
}
HandlesList[rd.GetString(0)] = co;
}
rd.Close();
}
}
catch (SqlException e)
{
var msg =
$"SQL exception : {e.Number}, {e.Message}";
SimpleLog.Error(msg);
return(false);
}
return(true);
}
static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List<string> extractedFiles, Action<string> updateMessage)
{
var sourcePath = Path.Combine(path, actionYaml.Value);
// Try as an absolute path
if (!File.Exists(sourcePath))
sourcePath = Platform.ResolvePath(actionYaml.Value);
using (var source = File.OpenRead(sourcePath))
{
foreach (var node in actionYaml.Nodes)
{
var targetPath = Platform.ResolvePath(node.Key);
if (File.Exists(targetPath))
{
Log.Write("install", "Skipping installed file " + targetPath);
continue;
}
var offsetNode = node.Value.Nodes.FirstOrDefault(n => n.Key == "Offset");
if (offsetNode == null)
{
Log.Write("install", "Skipping entry with missing Offset definition " + targetPath);
continue;
}
var lengthNode = node.Value.Nodes.FirstOrDefault(n => n.Key == "Length");
if (lengthNode == null)
{
Log.Write("install", "Skipping entry with missing Length definition " + targetPath);
continue;
}
var length = FieldLoader.GetValue<int>("Length", lengthNode.Value.Value);
source.Position = FieldLoader.GetValue<int>("Offset", offsetNode.Value.Value);
extractedFiles.Add(targetPath);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
Action<long> onProgress = null;
if (length < ShowPercentageThreshold)
updateMessage("Extracting " + displayFilename);
else
onProgress = b => updateMessage("Extracting " + displayFilename + " ({0}%)".F(100 * b / length));
using (var target = File.OpenWrite(targetPath))
{
Log.Write("install", "Extracting {0} -> {1}".F(sourcePath, targetPath));
if (type == ExtractionType.Blast)
{
Action<long, long> onBlastProgress = (read, _) =>
{
if (onProgress != null)
onProgress(read);
};
Blast.Decompress(source, target, onBlastProgress);
}
else
CopyStream(source, target, length, onProgress);
}
}
}
}
static void ExtractFromPackage(ExtractionType type, string path, MiniYaml actionYaml, List <string> extractedFiles, Action <string> updateMessage)
{
var sourcePath = Path.Combine(path, actionYaml.Value);
// Try as an absolute path
if (!File.Exists(sourcePath))
{
sourcePath = Platform.ResolvePath(actionYaml.Value);
}
using (var source = File.OpenRead(sourcePath))
{
foreach (var node in actionYaml.Nodes)
{
var targetPath = Platform.ResolvePath(node.Key);
if (File.Exists(targetPath))
{
Log.Write("install", "Skipping installed file " + targetPath);
continue;
}
var offsetNode = node.Value.Nodes.FirstOrDefault(n => n.Key == "Offset");
if (offsetNode == null)
{
Log.Write("install", "Skipping entry with missing Offset definition " + targetPath);
continue;
}
var lengthNode = node.Value.Nodes.FirstOrDefault(n => n.Key == "Length");
if (lengthNode == null)
{
Log.Write("install", "Skipping entry with missing Length definition " + targetPath);
continue;
}
var length = FieldLoader.GetValue <int>("Length", lengthNode.Value.Value);
source.Position = FieldLoader.GetValue <int>("Offset", offsetNode.Value.Value);
extractedFiles.Add(targetPath);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
var displayFilename = Path.GetFileName(Path.GetFileName(targetPath));
Action <long> onProgress = null;
if (length < ShowPercentageThreshold)
{
updateMessage("Extracting " + displayFilename);
}
else
{
onProgress = b => updateMessage("Extracting " + displayFilename + " ({0}%)".F(100 * b / length));
}
using (var target = File.OpenWrite(targetPath))
{
Log.Write("install", "Extracting {0} -> {1}".F(sourcePath, targetPath));
if (type == ExtractionType.Blast)
{
Action <long, long> onBlastProgress = (read, _) =>
{
if (onProgress != null)
{
onProgress(read);
}
};
Blast.Decompress(source, target, onBlastProgress);
}
else
{
CopyStream(source, target, length, onProgress);
}
}
}
}
}
private static extern void CalculateFeatureSizeCore(ExtractionType featureSetPurpose, ref Int32 featureSetSizeRecommended, ref Int32 featureSetSizeMinimum);
