public FileViewModel(CrawlingContext model, DispatcherQueue dispatcherQueue)
{
Model = model;
Resources = model.Resources
.Select(x => new ResourceViewModel(x, dispatcherQueue))
.Dispatch(dispatcherQueue);
var orderedResources = Resources.OrderBy(x => x.HttpStatus).ThenBy(x => x.URL);
var processedResources = orderedResources.Where(x => x.Status == ResourceStatus.Processed);
Nodes = new ObservableCollection <Node>
{
new Node("Resources", true, BitmapIcons.Folder, null,
new Node("By processing status", true, BitmapIcons.Folder, orderedResources,
EnumHelper <ResourceStatus> .Values
.Select(status => CreateNodeSplitByDomain(DescriptionExtractor.GetDescription(status), BitmapIcons.WorkStatuses.TryGetValue(status), orderedResources.Where(x => x.Status == status)))
.ToArray()),
new Node("By http status", true, BitmapIcons.Folder, processedResources,
from resource in processedResources
group resource by resource.HttpStatus into grp
orderby grp.Key
select new Node(DescriptionExtractor.GetDescription(grp.Key), false, BitmapIcons.GetImageFromHttpStatus(grp.Key), grp, SplitByDomain(grp))),
new Node("By bucket", true, BitmapIcons.Folder, null,
from resource in Resources
group resource by resource.CurrentBucket ?? "" into grp
where grp.Key != ""
orderby grp.Count descending, grp.Key
select new Node(grp.Key, false, BitmapIcons.Folder, grp)))
};
Model.PropertyChanged += Model_PropertyChanged;
}
public static DataTable GenerateErrorsReport(ICollection <Resource> resources)
{
DataTable table = new DataTable("Errors");
table.AddColumn("URL", typeof(string), 80);
table.AddColumn("Type", typeof(string), 15);
table.AddColumn("Line number", typeof(int), 15);
table.AddColumn("Message", typeof(string), 80);
table.AddColumn("Value", typeof(string), 80);
foreach (Resource resource in resources)
{
foreach (ResourceError error in resource.Errors)
{
table.Rows.Add(
resource.Url.AbsoluteUri,
DescriptionExtractor.GetDescription(error.Type),
error.Line,
error.Message,
error.Value);
}
}
return(table);
}
private string RenderEnums(ModuleFile file)
{
var endpointService = this.GetService <IEndpointService>();
var classes = endpointService.GetAllEndpointsClasses();
var enums = DescriptionExtractor.ExtractUniqueEnumsFromClasses(classes);
var enumValueItems = new Dictionary <string, string>();
DefaultContractResolver contractResolver = new DefaultContractResolver
{
NamingStrategy = new CamelCaseNamingStrategy(),
};
foreach (var enumItem in enums)
{
string serializedEnums = JsonConvert.SerializeObject(
EnumFunctions.GetEnumValueItems(enumItem.FullName),
new JsonSerializerSettings
{
ContractResolver = contractResolver,
Formatting = Formatting.Indented,
});
enumValueItems[enumItem.FullName] = JToken.Parse(serializedEnums).ToString();
}
return(file.RenderTemplate(new Dictionary <string, object>
{
{ "Enums", enums },
{ "EnumsValueItems", enumValueItems },
}));
}
public Unfluffer()
{
_htmlParser = new HtmlParser();
_languageExtractor = new LanguageExtractor();
_titleExtractor = new TitleExtractor();
_descriptionExtractor = new DescriptionExtractor();
_favIconExtractor = new FavIconExtractor();
_imageExtractor = new ImageExtractor();
_authorExtractor = new AuthorExtractor();
_canonicalExtractor = new CanonicalExtractor();
}
private string RenderEnums(ModuleFile file)
{
var endpointService = this.GetService <IEndpointService>();
var classes = endpointService.GetAllEndpointsClasses();
var enums = DescriptionExtractor.ExtractUniqueEnumsFromClasses(classes);
return(file.RenderTemplate(new Dictionary <string, object>
{
{ "Enums", enums },
}));
}
public override object ProvideValue(IServiceProvider serviceProvider)
{
EnumerationMember[] values;
if (!_cache.TryGetValue(EnumType, out values))
{
values = (
from object enumValue in Enum.GetValues(EnumType)
select new EnumerationMember
{
Value = enumValue,
Description = DescriptionExtractor.GetDescription(EnumType.GetField(enumValue.ToString()))
}).ToArray();
_cache.Add(EnumType, values);
}
return(values);
}
public object Convert(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture)
{
return(DescriptionExtractor.GetDescription(value));
}
public Description ExtractTest(string input)
{
return(DescriptionExtractor.Extract(input));
}
static ReportFieldNodes()
{
var nodeUrl = Create("Uri", (Uri uri) => uri,
Create("AbsolutePath", (Uri uri) => uri.AbsolutePath),
Create("AbsoluteUri", (Uri uri) => uri.AbsoluteUri),
Create("Authority", (Uri uri) => uri.Authority),
Create("Fragment", (Uri uri) => uri.Fragment),
Create("Host", (Uri uri) => uri.Host),
Create("PathAndQuery", (Uri uri) => uri.PathAndQuery),
Create("Query", (Uri uri) => uri.Query),
Create("Scheme", (Uri uri) => uri.Scheme));
var nodeReference =
Create("References", (ResourceReferenceCollection r) => r,
Create("First", (IEnumerable <ResourceReference> lst) => lst.FirstOrDefault()),
Create("Count", (IEnumerable <ResourceReference> lst) => lst.Count()));
NodeResource = Create("Resource", (ReportNodeInfos i) => i.Resource);
NodeReference = Create("Reference", (ReportNodeInfos i) => i.Reference);
NodeContent = Create("Content", (ReportNodeInfos i) => i.Content);
NodeError = Create("Error", (ReportNodeInfos i) => i.Error);
var referenceCollection = Create("ReferenceCollection", (IEnumerable <ResourceReference> coll) => coll,
Create("Count", (IEnumerable <ResourceReference> coll) => coll.Count()),
Create("First", (IEnumerable <ResourceReference> coll) => coll.FirstOrDefault()),
Create("First5", (IEnumerable <ResourceReference> coll) => new ReportArray(coll.Take(5).ToArray <object>())),
Create("First10", (IEnumerable <ResourceReference> coll) => new ReportArray(coll.Take(10).ToArray <object>())),
CreateAlias("Items", (IEnumerable <ResourceReference> coll) => new ReportArray(coll.ToArray <object>()), NodeReference));
NodeResource.AddNodes(
Create("ProcessingStatus", (Resource res) => res.Status),
Create("Status", (Resource res) => DescriptionExtractor.GetDescription(res.Status == ResourceStatus.Processed ? (object)res.HttpStatus : res.Status)),
Create("Content", (Resource res) => res.Content),
Create("Errors", (Resource res) => res.Errors),
Create("Redirections", (Resource res) => res,
CreateAlias("First", (Resource res) => res.References.FirstOrDefault(x => x.Type == ResourceReferenceTypes.Redirection), NodeReference),
Create <Resource, int>("Count", CountRedirections),
CreateAlias <Resource, Resource>("Final", FollowRedirections, NodeResource)),
Create("References", (Resource res) => res.References,
CreateAlias("All", (ResourceReferenceCollection res) => res, referenceCollection),
CreateAlias("Anchors", (ResourceReferenceCollection res) => res.Where(x => x.Type == ResourceReferenceTypes.Anchor), referenceCollection),
CreateAlias("Images", (ResourceReferenceCollection res) => res.Where(x => x.Type == ResourceReferenceTypes.Image), referenceCollection),
Create("Links", (ResourceReferenceCollection res) => res,
CreateAlias("Canonicals", (ResourceReferenceCollection res) => res.Where(x => x.SubType == ReferenceSubType.Canonical), referenceCollection),
CreateAlias("ShortcutIcons", (ResourceReferenceCollection res) => res.Where(x => x.SubType == ReferenceSubType.ShortcutIcon), referenceCollection),
CreateAlias("StyleSheets", (ResourceReferenceCollection res) => res.Where(x => x.SubType == ReferenceSubType.StyleSheet), referenceCollection)),
CreateAlias("Scripts", (ResourceReferenceCollection res) => res.Where(x => x.Type == ResourceReferenceTypes.Script), referenceCollection)),
CreateAlias("Referers", (Resource res) => res.ReferencedBy, referenceCollection),
Create("Request", (Resource res) => res,
Create("Behavior", (Resource res) => res.Behavior),
Create("TimeStart", (Resource res) => res.TimeStart),
CreateAlias("Url", (Resource res) => res.Url, nodeUrl)),
Create("Response", (Resource res) => res,
Create("CompressedSize", (Resource res) => res.CompressedSize),
Create("ContentType", (Resource res) => res.ContentType),
Create("Headers", (Resource res) => res.Headers,
Create("ContentEncoding", (ResourceHeaders h) => h.TryGetValue("content-encoding")),
Create("ContentType", (ResourceHeaders h) => h.TryGetValue("content-type")),
Create("CacheControl", (ResourceHeaders h) => h.TryGetValue("cache-control")),
Create("HasETAG", (ResourceHeaders h) => !String.IsNullOrWhiteSpace(h.TryGetValue("ETag"))),
Create("LastModified", (ResourceHeaders h) => TryParseDate(h.TryGetValue("last-modified"))),
Create("Expires", (ResourceHeaders h) => TryParseDate(h.TryGetValue("expires")))),
Create("HttpStatus", (Resource res) => res.HttpStatus),
Create("Size", (Resource res) => res.Size),
Create("CompressedSize", (Resource res) => res.CompressedSize),
Create("TimeLoading", (Resource res) => res.TimeLoading),
Create("TimeProcessing", (Resource res) => res.TimeProcessing),
Create("ViewStateSize", (Resource res) => res.ViewStateSize ?? 0)));
NodeReference.AddNodes(
CreateAlias("Source", (ResourceReference r) => r.Source, NodeResource),
CreateAlias("Target", (ResourceReference r) => r.Target, NodeResource),
Create("Type", (ResourceReference r) => r.Type),
Create("SubType", (ResourceReference r) => r.SubType),
Create("RawUrl", (ResourceReference r) => r.Url.OriginalString),
Create("Count", (ResourceReference r) => r.Count),
Create("IsRecursive", (ResourceReference r) => r.Source == r.Target || r.Source == FollowRedirections(r.Target)));
NodeContent.AddNodes(
Create("Key", (ResourceContent c) => c.Key),
Create("Value", (ResourceContent c) => c.Value));
NodeError.AddNodes(
Create("Type", (ResourceError e) => e.Type),
Create("Message", (ResourceError e) => e.Message),
Create("Value", (ResourceError e) => e.Value),
Create("Line", (ResourceError e) => e.Line));
Root = Create("Root", (ReportNodeInfos i) => i, NodeResource, NodeReference, NodeContent, NodeError);
}
public static IEnumerable <string> ConvertToJsFetchRequest(AccountStatementResponse[] accountStatements)
{
var transactions = accountStatements
.Select(
statement => new
{
statement,
StatementType = statement.Type.StringToEnum <AccountStatementType>()
})
.Where(pair => !Constants.IgnoringAccountStatementType.Contains(pair.StatementType))
.Select(
pair =>
{
var statement = pair.statement;
var description = DescriptionExtractor.Extract(statement.Description);
if (description.BorrowerName == null)
{
throw new NotSupportedException(
$"{nameof(Description.BorrowerName)} must be provided for {statement.Type}");
}
var comment = new StringBuilder().Append($"{statement.Title}: проект {description.ProjectId}")
.AppendIf(description.TimePeriodInDays.HasValue, $" срок {description.TimePeriodInDays} д.")
.AppendIf(description.Percent.HasValue, $" под {description.Percent}%");
var transaction = new Transaction
{
Category = Constants.ZeroValue,
Comment = comment.ToString(),
Date = statement.CreatedAt.ToString("dd.MM.yyyy"),
Payee = description.BorrowerName.Replace("\"", null)
};
string GetFormattedAmount() =>
Math.Abs(statement.Amount).ToString(CultureInfo.InvariantCulture);
var statementType = statement.Type.StringToEnum <AccountStatementType>();
switch (statementType)
{
case AccountStatementType.RefundMain:
if (statement.Amount < 0)
{
throw new NotSupportedException(
$"Not expected that Amount is negative: {new { statement.Amount, statement.Type }.ToJson()}");
}
transaction.AccountIncome = Constants.Wallets.МодульДеньгиRub;
transaction.AccountOutcome = Constants.Wallets.Долг;
transaction.Outcome = transaction.Income = GetFormattedAmount();
return(transaction);
case AccountStatementType.Fund:
if (statement.Amount > 0)
{
throw new NotSupportedException(
$"Not expected that Amount is positive: {new { statement.Amount, statement.Type }.ToJson()}");
}
transaction.AccountIncome = Constants.Wallets.Долг;
transaction.AccountOutcome = Constants.Wallets.МодульДеньгиRub;
transaction.Outcome = transaction.Income = GetFormattedAmount();
return(transaction);
case AccountStatementType.CessionCommission:
transaction.TagGroups = new[]
{
Constants.Categories.Комиссия
};
break;
case AccountStatementType.RefundPercentage:
case AccountStatementType.RefundPenaltyPercentage:
transaction.TagGroups = new[]
{
Constants.Categories.ДолгПроценты
};
break;
default:
throw new NotSupportedException(statement.Type);
}
transaction.AccountIncome = transaction.AccountOutcome = Constants.Wallets.МодульДеньгиRub;
if (statement.Amount < 0)
{
transaction.Outcome = GetFormattedAmount();
transaction.Income = Constants.ZeroValue;
}
else
{
transaction.Income = GetFormattedAmount();
transaction.Outcome = Constants.ZeroValue;
}
return(transaction);
}).ToArray();
foreach (var transaction in transactions)
{
yield return(WrapToFetch(
new[]
{
transaction
}));
}
}
