public async Task <IActionResult> PutOffer(string id, RawProperty offer)
{
if (id != offer.Id)
{
return(BadRequest());
}
_context.Entry(offer).State = EntityState.Modified;
try
{
await _context.SaveChangesAsync();
}
catch (DbUpdateConcurrencyException)
{
if (!OfferExists(id))
{
return(NotFound());
}
else
{
throw;
}
}
return(NoContent());
}
public void PopulateContentArea_WithMultipleFunctions_ShouldAddContentFragmentsForEachFunctionToPropertyValue()
{
// Arrange
var subject = CreateSubject(fragmentBuilder: CreateFragmentBuilder());
var property = new RawProperty();
var guids = new[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() };
var functions = new[]
{
new ComposerContentFunction {
Guid = guids[0]
},
new ComposerContentFunction {
Guid = guids[1]
},
new ComposerContentFunction {
Guid = guids[2]
}
};
// Act
subject.PopulateContentArea(property, functions);
// Assert
for (int i = 0; i < guids.Length; i++)
{
Assert.IsTrue(property.Value.Contains(guids[i].ToString()));
}
}
private void OnContentImporting(DataImporter importing, ContentImportingEventArgs args)
{
RawProperty property = args.TransferContentData.RawContentData.Property.Find(p => p.Name.Equals("PageName"));
if (property == null)
{
property = args.TransferContentData.RawContentData.Property[0];
}
_log.Debug("Content Import: {0} = {1}", property.Name, property.Value);
}
public void TransformFunctionPickerProperty_ShouldChangeTheTypeToContentReference()
{
// Arrange
var subject = CreateSubject();
var property = new RawProperty { Type = PropertyDataType.String, TypeName = ComposerPropertyTypes.FunctionPicker, AssemblyName = "Dropit.Extension" };
// Act
subject.TransformFunctionPickerProperty(property);
// Assert
Assert.AreEqual(PropertyDataType.ContentReference, property.Type);
Assert.IsNull(property.TypeName);
Assert.IsNull(property.AssemblyName);
}
public void PopulateContentArea_ShouldAddContentFragmentsToPropertyValue()
{
// Arrange
var subject = CreateSubject(fragmentBuilder: CreateFragmentBuilder());
var property = new RawProperty();
var guid = Guid.NewGuid();
var functions = new[] { new ComposerContentFunction { Guid = guid } };
// Act
subject.PopulateContentArea(property, functions);
// Assert
Assert.IsFalse(property.IsNull);
Assert.AreEqual(guid.ToString(), property.Value);
}
public void TransformFunctionPickerProperty_ShouldChangeTheTypeToContentReference()
{
// Arrange
var subject = CreateSubject();
var property = new RawProperty {
Type = PropertyDataType.String, TypeName = ComposerPropertyTypes.FunctionPicker, AssemblyName = "Dropit.Extension"
};
// Act
subject.TransformFunctionPickerProperty(property);
// Assert
Assert.AreEqual(PropertyDataType.ContentReference, property.Type);
Assert.IsNull(property.TypeName);
Assert.IsNull(property.AssemblyName);
}
public void PopulateContentArea_ShouldAddContentFragmentsToPropertyValue()
{
// Arrange
var subject = CreateSubject(fragmentBuilder: CreateFragmentBuilder());
var property = new RawProperty();
var guid = Guid.NewGuid();
var functions = new[] { new ComposerContentFunction {
Guid = guid
} };
// Act
subject.PopulateContentArea(property, functions);
// Assert
Assert.IsFalse(property.IsNull);
Assert.AreEqual(guid.ToString(), property.Value);
}
public virtual void TransformFunctionPickerProperty(RawProperty functionPickerProperty)
{
// Change type to ContentReference
functionPickerProperty.Type = PropertyDataType.ContentReference;
functionPickerProperty.TypeName = null;
functionPickerProperty.AssemblyName = null;
var data = _composerSerializer.Deserialize<FunctionPickerData>(functionPickerProperty.Value);
// Convert the Value to an ExportableLink
if (data != null && data.FunctionLink != null)
{
functionPickerProperty.Value = _exportLinkResolver.GetExportableLink(data.FunctionLink.Guid, data.PageLanguage);
functionPickerProperty.IsNull = string.IsNullOrEmpty(functionPickerProperty.Value);
}
else
{
functionPickerProperty.IsNull = true;
}
}
public virtual void TransformFunctionPickerProperty(RawProperty functionPickerProperty)
{
// Change type to ContentReference
functionPickerProperty.Type = PropertyDataType.ContentReference;
functionPickerProperty.TypeName = null;
functionPickerProperty.AssemblyName = null;
var data = _composerSerializer.Deserialize <FunctionPickerData>(functionPickerProperty.Value);
// Convert the Value to an ExportableLink
if (data != null && data.FunctionLink != null)
{
functionPickerProperty.Value = _exportLinkResolver.GetExportableLink(data.FunctionLink.Guid, data.PageLanguage);
functionPickerProperty.IsNull = string.IsNullOrEmpty(functionPickerProperty.Value);
}
else
{
functionPickerProperty.IsNull = true;
}
}
public virtual void PopulateContentArea(RawProperty contentAreaProperty, IEnumerable <ComposerContentFunction> composerFunctions)
{
if (composerFunctions == null || !composerFunctions.Any())
{
contentAreaProperty.IsNull = true;
return;
}
// Populate the ContentArea property from the ContentFunction data
var contentArea = new ContentArea();
foreach (var block in composerFunctions)
{
var mappedGuid = GetMappedGuid(block.Guid);
var fragment = _contentFragmentBuilder.CreateFragment(mappedGuid, block.PersonalizationGroup, block.VisitorGroups);
contentArea.Fragments.Add(fragment);
}
contentAreaProperty.Value = contentArea.ToInternalString();
contentAreaProperty.IsNull = false;
}
public void PopulateContentArea_WithMultipleFunctions_ShouldAddContentFragmentsForEachFunctionToPropertyValue()
{
// Arrange
var subject = CreateSubject(fragmentBuilder: CreateFragmentBuilder());
var property = new RawProperty();
var guids = new[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() };
var functions = new[]
{
new ComposerContentFunction { Guid = guids[0] },
new ComposerContentFunction { Guid = guids[1] },
new ComposerContentFunction { Guid = guids[2] }
};
// Act
subject.PopulateContentArea(property, functions);
// Assert
for (int i = 0; i < guids.Length; i++)
{
Assert.IsTrue(property.Value.Contains(guids[i].ToString()));
}
}
public virtual void PopulateContentArea(RawProperty contentAreaProperty, IEnumerable<ComposerContentFunction> composerFunctions)
{
composerFunctions = composerFunctions.Where(f => f.Guid != Guid.Empty);
if (composerFunctions == null || !composerFunctions.Any())
{
contentAreaProperty.IsNull = true;
return;
}
// Populate the ContentArea property from the ContentFunction data
var contentArea = new ContentArea();
foreach (var block in composerFunctions)
{
var mappedGuid = GetMappedGuid(block.Guid);
var fragment = _contentFragmentBuilder.CreateFragment(mappedGuid, block.PersonalizationGroup, block.VisitorGroups);
contentArea.Fragments.Add(fragment);
}
contentAreaProperty.Value = contentArea.ToInternalString();
contentAreaProperty.IsNull = false;
}
private void Draw()
{
if (ComponentSpecification == null)
{
return;
}
Cursor = Cursors.WaitCursor;
Font boldFont = new Font(Font, FontStyle.Bold);
MainShape = new RawComponent(shapeCanvas1, ComponentSpecification.Name, boldFont, ComponentSpecification);
MainShape.Categories.Clear();
RawCategory cat = new RawCategory("Properties", Font, MainShape);
MainShape.Categories.Add(cat);
foreach (var x in ComponentSpecification.Properties)
{
//RawProperty prop = new RawProperty(string.Format("{0} [{1}]", x.Name, x.Type), x);
RawProperty prop = new RawProperty(x.Name, x);
cat.Properties.Add(prop);
}
//centreShape.Icon = EntityImage;
//centreShape.Enter += new EventHandler(centreShape_Enter);
//centreShape.Leave += new EventHandler(centreShape_Leave);
Font boldUnderlineFont = new Font(Font, FontStyle.Bold | FontStyle.Underline);
List <RawShape> topLevelShapes = new List <RawShape>();
List <RawShape> rightAlignedShapes = new List <RawShape>();
#region References
foreach (ComponentImpl component in ComponentSpecification.ImplementedComponents)
{
RawEntity referencedEntityShape;
CustomLineCap startCap;
CustomLineCap endCap;
referencedEntityShape = new RawEntity(shapeCanvas1, component.ParentEntity.Name, boldFont, component.ParentEntity);
//string end1 = ArchAngel.Interfaces.Cardinality.One.Equals(reference.Cardinality2) ? "1" : "m";
//startCap = ArchAngel.Interfaces.Cardinality.One.Equals(reference.Cardinality2) ? LineCaps.One : LineCaps.Many;
//endCap = ArchAngel.Interfaces.Cardinality.One.Equals(reference.Cardinality1) ? LineCaps.One : LineCaps.Many;
referencedEntityShape.OriginatingLineStyle.EndTextDataMember = "Name";
//referencedEntityShape.OriginatingLineStyle.EndTextDataMember = "End2Name";
//referencedEntityShape.OriginatingLineStyle.StartImageClick += new LinkLine.MouseEndDelegate(OriginatingLineStyle_StartImageClick);
//referencedEntityShape.OriginatingLineStyle.EndImageClick += new LinkLine.MouseEndDelegate(OriginatingLineStyle_EndImageClick);
//referencedEntityShape.OriginatingLineStyle.MouseOverEnd1 += new LinkLine.MouseOverEndDelegate(OriginatingLineStyle_MouseOverEnd2);
//referencedEntityShape.OriginatingLineStyle.MouseOverEnd2 += new LinkLine.MouseOverEndDelegate(OriginatingLineStyle_MouseOverEnd1);
referencedEntityShape.Icon = EntityImage;
referencedEntityShape.OriginatingLineStyle.LineStyle = DashStyle.Solid;
//referencedEntityShape.OriginatingLineStyle.StartCap = startCap;
//referencedEntityShape.OriginatingLineStyle.EndCap = endCap;
referencedEntityShape.OriginatingLineStyle.DataObject = component;
referencedEntityShape.OriginatingLineStyle.ForeColor = Color.White;
referencedEntityShape.OriginatingLineStyle.DefaultEndText = "";
//referencedEntityShape.OriginatingLineStyle.StartImage = EditImage;
//referencedEntityShape.OriginatingLineStyle.EndImage = EditImage;
referencedEntityShape.OriginatingLineStyle.MiddleImage = InfoImage;
referencedEntityShape.OriginatingLineStyle.MiddleImageClick += new MouseEventHandler(OriginatingLineStyle_MiddleImageClick);
referencedEntityShape.MouseDoubleClick += new MouseEventHandler(referencedEntityShape_MouseDoubleClick);
referencedEntityShape.MouseClick += new MouseEventHandler(referencedEntityShape_MouseClick);
referencedEntityShape.Cursor = Cursors.Hand;
topLevelShapes.Add(referencedEntityShape);
}
// Add empty reference
RawShape emptyReference = new RawShape(shapeCanvas1, "Add entity...", boldUnderlineFont)
{
BackColor1 = Color.White,
BackColor2 = Color.White,
BorderColor = Color.Gray,
ForeColor = Color.Gray,
FocusForeColor = Color.Blue,
FocusBackColor1 = Color.WhiteSmoke,
FocusBackColor2 = Color.White,
FocusBorderColor = Color.DarkGray,
Cursor = Cursors.Hand,
OriginatingLineStyle = null, // new LinkLine(boldFont, DashStyle.Dot, "", "", "", LineCaps.None, LineCaps.SolidArrow),
Tag = null
};
//emptyReference.OriginatingLineStyle.ForeColor = Color.White;
emptyReference.MouseClick += new MouseEventHandler(emptyReference_MouseClick);
topLevelShapes.Add(emptyReference);
#endregion
shapeCanvas1.SwimLane1 = new ShapeCanvas.SwimLaneStyle(Color.Gray, Color.Black, Color.White, 180F, "Used By (Entities)", ShapeCanvas.SwimLaneStyle.Styles.Fill);
//shapeCanvas1.SwimLane3 = new ShapeCanvas.SwimLaneStyle(Color.FromArgb(79, 124, 205), Color.Black, Color.White, 0F, "Mapped Tables", ShapeCanvas.SwimLaneStyle.Styles.Fill);
shapeCanvas1.BackColor = this.BackColor;
shapeCanvas1.Height = this.Height;
shapeCanvas1.KeepMainShapeCentered = true;
shapeCanvas1.DrawThreeLayerHorizontal(MainShape, topLevelShapes, rightAlignedShapes, KeepMainShapeFull);
Cursor = Cursors.Default;
}
public async Task <ActionResult <IEnumerable <RawProperty> > > Get() //(int fromSize, int toSize)
{
int fromSize = 10;
int toSize = 1000;
var properties = new List <RawProperty>();
var parser = new HtmlParser();
var handler = new HttpClientHandler {
AllowAutoRedirect = false,
};
var client = new HttpClient(handler);
var floorsRegex = new Regex(@"(?<floor>[0-9]+)[^\d]+(?<all>[0-9]+)", RegexOptions.Compiled);
var typeAndInfoRegex = new Regex(@"^(?<type>[^,]+)([,\s]+(?<year>[0-9]+))?", RegexOptions.Compiled);
// 10 => 1000
for (var size = fromSize; size <= toSize; size++)
{
Console.Write($"Area {size}: ");
var formData =
$"act=3&rub=1&rub_pub_save=1&topmenu=2&actions=1&f0=127.0.0.1&f1=1&f2=&f3=&f4=1&f7=1%7E2%7E3%7E4%7E5%7E6%7E8%7E&f28=&f29=&f43=&f44=&f30=EUR&f26={size}&f27={size}&f41=1&f31=&f32=&f38=%E3%F0%E0%E4+%D1%EE%F4%E8%FF&f42=&f39=&f40=&fe3=&fe4=&f45=&f46=&f51=&f52=&f33=&f34=&f35=&f36=&f37=&fe2=1";
//$"act=3&rub=1&rub_pub_save=1&topmenu=2&actions=1&f0=127.0.0.1&f1=1&f2=&f3=&f4=1&f7=1%7E2%7E3%7E4%7E5%7E6%7E8%7E&f28=&f29=&f43=&f44=&f30=EUR&f26={size}&f27={size}&f41=1&f31=&f32=&f38=%E3%F0%E0%E4+{town}&f42=&f39=&f40=&fe3=&fe4=&f45=&f46=&f51=&f52=&f33=&f34=&f35=&f36=&f37=&fe2=1";
var response = await client.PostAsync(
"https://www.imot.bg/pcgi/imot.cgi",
new StringContent(formData, Encoding.UTF8, "application/x-www-form-urlencoded"));
var firstPageUrl = response.Headers.Location;
for (var page = 1; page <= 26; page++)
{
var pageUrl = firstPageUrl.ToString().Replace("&f1=1", $"&f1={page}");
var pageHtml = await GetHtml(pageUrl);
var pageDocument = await parser.ParseDocumentAsync(pageHtml);
var listItems = pageDocument.QuerySelectorAll("a.photoLink").Where(
x => x.Attributes["href"]?.Value?.Contains("pcgi/imot.cgi?act=5&adv=") == true).ToList();
if (!listItems.Any())
{
break;
}
foreach (var listItem in listItems)
{
var url = "https:" + listItem.Attributes["href"].Value;
var html = await GetHtml(url);
var document = await parser.ParseDocumentAsync(html);
var district = document.QuerySelectorAll("table")[2].QuerySelectorAll("tr td div")[1].TextContent;
if (district?.Contains("<br>") == true)
{
var indexOfBr = district.IndexOf("<br>", StringComparison.InvariantCulture);
district = district.Substring(0, indexOfBr).Trim();
}
var pic = document.QuerySelector("img.big").Attributes["src"].Value;
//Console.WriteLine(pic);
var floorInfoString = document.QuerySelectorAll("ul.imotData li")[3].TextContent;;
var floorMatch = floorsRegex.Match(floorInfoString);
var typeAndInfoString = document.QuerySelector("ul.imotData").Children.Last().TextContent;
var typeAndInfoMatch = typeAndInfoRegex.Match(typeAndInfoString);
var property = new RawProperty
{
Url = url,
Size = size,
District = district,
Type = typeAndInfoString.Split(",")[0],
Floor =
floorMatch.Success ? floorMatch.Groups["floor"].Value.ToInteger() : 0,
TotalFloors =
floorMatch.Success ? floorMatch.Groups["all"].Value.ToInteger() : 0,
Price =
document.QuerySelector("span#cena")?.TextContent
?.Replace(" EUR", string.Empty)?.ToInteger() ?? 0,
Year = typeAndInfoMatch.Success && typeAndInfoMatch.Groups["year"].Success
? typeAndInfoMatch.Groups["year"].Value.ToInteger()
: 0,
BuildingType =
document.QuerySelectorAll("table")[2].QuerySelectorAll("tr td div h1")[0].TextContent.Replace("Продава ", string.Empty),
CreatedAt = DateTime.UtcNow,
Pic = pic,
};
if (!OfferExists(property.Url) && property.TotalFloors != 0 && property.Year != 0 && property.Price != 0)
//if (property.TotalFloors != 0 && property.Year != 0 && property.Price != 0)
{
properties.Add(property);
await context.AddAsync(property);
await context.SaveChangesAsync();
}
}
Console.Write($"{page}({properties.Count}), ");
}
//await context.AddRangeAsync(properties);
//await context.SaveChangesAsync();
Console.WriteLine($" => Total: {properties.Count}");
}
//await context.AddRangeAsync(properties);
// var result = await context.SaveChangesAsync();
return(properties.OrderByDescending(x => x.CreatedAt)
.Take(5).ToList());
async Task <string> GetHtml(string pageUrl)
{
var pageResponse = await client.GetAsync(pageUrl);
var byteContent = await pageResponse.Content.ReadAsByteArrayAsync();
var html = Encoding.GetEncoding("windows-1251").GetString(byteContent);
return(html);
}
}
public static void Write(Stream output, RawProperty instance, Endian endian)
{
output.WriteValueU32(instance.NameHash, endian);
output.WriteBytes(instance.Data ?? (_DummyData ?? (_DummyData = new byte[4]))); // lol
output.WriteValueU8((byte)instance.Type);
}
public void Deserialize(Stream input)
{
var basePosition = input.Position;
var magic = input.ReadValueU32(Endian.Little);
if (magic != Signature && magic.Swap() != Signature)
{
throw new FormatException();
}
var endian = magic == Signature ? Endian.Little : Endian.Big;
var version = input.ReadValueU32(endian);
if (version != 1)
{
throw new FormatException();
}
var rawRootNode = RawNode.Read(input, endian);
var rootNode = new Node();
rootNode.NameHash = rawRootNode.NameHash;
var instanceQueue = new Queue <Tuple <Node, RawNode> >();
instanceQueue.Enqueue(new Tuple <Node, RawNode>(rootNode, rawRootNode));
var propertyQueue = new Queue <Tuple <Node, RawProperty[]> >();
while (instanceQueue.Count > 0)
{
var item = instanceQueue.Dequeue();
var node = item.Item1;
var rawNode = item.Item2;
input.Position = basePosition + rawNode.DataOffset;
var rawProperties = new RawProperty[rawNode.PropertyCount];
for (int i = 0; i < rawNode.PropertyCount; i++)
{
rawProperties[i] = RawProperty.Read(input, endian);
}
input.Position = basePosition +
(rawNode.DataOffset + (RawProperty.Size * rawNode.PropertyCount)).Align(4);
for (int i = 0; i < rawNode.InstanceCount; i++)
{
var rawChildNode = RawNode.Read(input, endian);
var childNode = new Node();
childNode.NameHash = rawChildNode.NameHash;
node.Children.Add(childNode);
instanceQueue.Enqueue(new Tuple <Node, RawNode>(childNode, rawChildNode));
}
propertyQueue.Enqueue(new Tuple <Node, RawProperty[]>(node, rawProperties));
}
while (propertyQueue.Count > 0)
{
var item = propertyQueue.Dequeue();
var node = item.Item1;
var rawProperties = item.Item2;
foreach (var rawProperty in rawProperties)
{
var variant = VariantFactory.GetVariant(rawProperty.Type);
if (variant.IsPrimitive == true)
{
using (var temp = new MemoryStream(rawProperty.Data, false))
{
variant.Deserialize(temp, endian);
if (temp.Position != temp.Length)
{
throw new InvalidOperationException();
}
}
}
else
{
if (rawProperty.Data.Length != 4)
{
throw new InvalidOperationException();
}
uint offset;
using (var temp = new MemoryStream(rawProperty.Data, false))
{
offset = temp.ReadValueU32(endian);
}
input.Position = basePosition + offset;
variant.Deserialize(input, endian);
}
node.Properties.Add(rawProperty.NameHash, (IVariant)variant);
}
}
this._Root = rootNode;
}
public void Serialize(Stream output)
{
var endian = this._Endian;
if (this._Root == null)
{
output.WriteValueU32(Signature, endian);
output.WriteValueU32(1, endian); // version
new RawNode(0, 8 + RawNode.Size, 0, 0).Write(output, endian);
return;
}
var stringOffsets = new Dictionary <string, uint>();
using (var data = new MemoryStream())
{
data.WriteValueU32(Signature, endian);
data.WriteValueU32(1, endian); // version
var rawNodes = new List <Tuple <long, RawNode> >();
var queue = new Queue <Tuple <long, Node> >();
queue.Enqueue(new Tuple <long, Node>(data.Position, this._Root));
data.Position += RawNode.Size; // root node size
while (queue.Count > 0)
{
var tuple = queue.Dequeue();
var rawPosition = tuple.Item1;
var node = tuple.Item2;
var propertyPosition = data.Position;
var childPosition = (propertyPosition + node.Properties.Count * RawProperty.Size).Align(4);
var propertyDataPosition = childPosition + (node.Children.Count * RawNode.Size);
var rawNode = new RawNode(node.NameHash,
(uint)propertyPosition,
(ushort)node.Properties.Count,
(ushort)node.Children.Count);
rawNodes.Add(new Tuple <long, RawNode>(rawPosition, rawNode));
data.Position = propertyDataPosition;
var rawProperties = new List <RawProperty>();
foreach (var kv in node.Properties.OrderBy(kv => kv.Key))
{
var rawVariant = (IRawVariant)kv.Value;
RawProperty rawProperty;
if (rawVariant.IsPrimitive == true)
{
var bytes = new byte[4];
using (var temp = new MemoryStream(bytes))
{
rawVariant.Serialize(temp, endian);
}
rawProperty = new RawProperty(kv.Key, bytes, rawVariant.Type);
}
else if (rawVariant is Variants.StringVariant)
{
var stringVariant = (Variants.StringVariant)rawVariant;
uint dataOffset;
if (stringOffsets.ContainsKey(stringVariant.Value) == false)
{
if (rawVariant.Alignment > 0)
{
data.Position = data.Position.Align(rawVariant.Alignment);
}
dataOffset = (uint)data.Position;
rawVariant.Serialize(data, endian);
stringOffsets.Add(stringVariant.Value, dataOffset);
}
else
{
dataOffset = stringOffsets[stringVariant.Value];
}
var bytes = new byte[4];
using (var temp = new MemoryStream(bytes))
{
temp.WriteValueU32(dataOffset, endian);
}
rawProperty = new RawProperty(kv.Key, bytes, rawVariant.Type);
}
else
{
if (rawVariant.Alignment > 0)
{
data.Position = data.Position.Align(rawVariant.Alignment);
}
var dataOffset = (uint)data.Position;
rawVariant.Serialize(data, endian);
var bytes = new byte[4];
using (var temp = new MemoryStream(bytes))
{
temp.WriteValueU32(dataOffset, endian);
}
rawProperty = new RawProperty(kv.Key, bytes, rawVariant.Type);
}
rawProperties.Add(rawProperty);
}
var childDataPosition = data.Position.Align(4);
data.Position = propertyPosition;
foreach (var rawProperty in rawProperties)
{
rawProperty.Write(data, endian);
}
data.Position = childDataPosition;
foreach (var child in node.Children.OrderBy(c => c.NameHash))
{
queue.Enqueue(new Tuple <long, Node>(childPosition, child));
childPosition += RawNode.Size;
}
}
foreach (var tuple in rawNodes)
{
var rawPosition = tuple.Item1;
var rawNode = tuple.Item2;
data.Position = rawPosition;
rawNode.Write(data, endian);
}
data.Flush();
data.Position = 0;
output.WriteFromStream(data, data.Length);
}
}
private void WriteNode(Stream data, Node node, Dictionary <string, uint> stringOffsets, List <Tuple <long, RawNode> > rawNodes)
{
var endian = this._Endian;
var propertyPosition = data.Position;
var childPosition = (propertyPosition + node.Properties.Count * RawProperty.Size).Align(4);
var propertyDataPosition = childPosition + (node.Children.Count * RawNode.Size);
data.Position = propertyDataPosition;
var rawProperties = new List <RawProperty>();
foreach (var kv in node.Properties.OrderBy(kv => kv.Key))
{
var rawVariant = (IRawVariant)kv.Value;
RawProperty rawProperty;
if (rawVariant.IsPrimitive == true)
{
var bytes = new byte[4];
using (var temp = new MemoryStream(bytes))
{
rawVariant.Serialize(temp, endian);
}
rawProperty = new RawProperty(kv.Key, bytes, rawVariant.Type);
}
else if (rawVariant is Variants.StringVariant)
{
var stringVariant = (Variants.StringVariant)rawVariant;
uint dataOffset;
if (stringOffsets.ContainsKey(stringVariant.Value) == false)
{
if (rawVariant.Alignment > 0)
{
data.Position = data.Position.Align(rawVariant.Alignment);
}
dataOffset = (uint)data.Position;
rawVariant.Serialize(data, endian);
stringOffsets.Add(stringVariant.Value, dataOffset);
}
else
{
dataOffset = stringOffsets[stringVariant.Value];
}
var bytes = new byte[4];
using (var temp = new MemoryStream(bytes))
{
temp.WriteValueU32(dataOffset, endian);
}
rawProperty = new RawProperty(kv.Key, bytes, rawVariant.Type);
}
else
{
if (rawVariant.Alignment > 0)
{
data.Position = data.Position.Align(rawVariant.Alignment);
}
var dataOffset = (uint)data.Position;
rawVariant.Serialize(data, endian);
var bytes = new byte[4];
using (var temp = new MemoryStream(bytes))
{
temp.WriteValueU32(dataOffset, endian);
}
rawProperty = new RawProperty(kv.Key, bytes, rawVariant.Type);
}
rawProperties.Add(rawProperty);
}
var childDataPosition = data.Position.Align(4);
data.Position = propertyPosition;
foreach (var rawProperty in rawProperties.OrderBy(kv => kv.NameHash))
{
rawProperty.Write(data, endian);
}
data.Position = childDataPosition;
foreach (var child in node.Children.OrderBy(c => c.NameHash))
{
var rawNode = new RawNode(child.NameHash,
(uint)data.Position,
(ushort)child.Properties.Count,
(ushort)child.Children.Count);
rawNodes.Add(new Tuple <long, RawNode>(childPosition, rawNode));
childPosition += RawNode.Size;
WriteNode(data, child, stringOffsets, rawNodes);
}
}
private string RawPropertyToTypeString(RawProperty propertyDefinition)
{
switch (propertyDefinition.DataType)
{
case DataType.Boolean: return(typeof(bool).Name);
case DataType.SByte: return(typeof(SByte).Name);
case DataType.Int16: return(typeof(Int16).Name);
case DataType.Int32: return(typeof(Int32).Name);
case DataType.Int64: return(typeof(Int64).Name);
case DataType.Byte: return(typeof(Byte).Name);
case DataType.UInt16: return(typeof(UInt16).Name);
case DataType.UInt32: return(typeof(UInt32).Name);
case DataType.UInt64: return(typeof(UInt64).Name);
case DataType.Single: return(typeof(Single).Name);
case DataType.Double: return(typeof(Double).Name);
case DataType.Guid: return(typeof(Guid).Name);
case DataType.String: return(typeof(DataCoreString).Name);
case DataType.Locale: return(typeof(DataCoreLocale).Name);
case DataType.Enum:
{
var enumDefinition = RawDatabase.enumDefinitions[propertyDefinition.DefinitionIndex];
var enumName = RawDatabase.textBlock.GetString(enumDefinition.NameOffset);
return(enumName);
}
case DataType.Class:
{
var structureDefinition = RawDatabase.structureDefinitions[propertyDefinition.DefinitionIndex];
var structureName = RawDatabase.textBlock.GetString(structureDefinition.NameOffset);
return(structureName);
}
case DataType.StrongPointer:
{
var structureDefinition = RawDatabase.structureDefinitions[propertyDefinition.DefinitionIndex];
var structureName = RawDatabase.textBlock.GetString(structureDefinition.NameOffset);
return($"DataCoreStrongPointer<{structureName}>");
}
case DataType.WeakPointer:
{
var structureDefinition = RawDatabase.structureDefinitions[propertyDefinition.DefinitionIndex];
var structureName = RawDatabase.textBlock.GetString(structureDefinition.NameOffset);
return($"DataCoreWeakPointer<{structureName}>");
}
case DataType.Reference:
{
var structureDefinition = RawDatabase.structureDefinitions[propertyDefinition.DefinitionIndex];
var structureName = RawDatabase.textBlock.GetString(structureDefinition.NameOffset);
return($"DataCoreRecord<{structureName}>");
}
default:
throw new NotImplementedException();
}
}
public async Task <IEnumerable <RawProperty> > GatherData(int fromSize, int toSize)
{
var properties = new List <RawProperty>();
var parser = new HtmlParser();
var handler = new HttpClientHandler {
AllowAutoRedirect = false,
};
var client = new HttpClient(handler);
var floorsRegex = new Regex(@"(?<floor>[0-9]+)[^\d]+(?<all>[0-9]+)", RegexOptions.Compiled);
var typeAndInfoRegex = new Regex(@"^(?<type>[^,]+)([,\s]+(?<year>[0-9]+))?", RegexOptions.Compiled);
// 10 => 1000
for (var size = fromSize; size <= toSize; size++)
{
Console.Write($"Area {size}: ");
var formData =
$"act=3&rub=1&rub_pub_save=1&topmenu=2&actions=1&f0=127.0.0.1&f1=1&f2=&f3=&f4=1&f7=1%7E2%7E3%7E4%7E5%7E6%7E8%7E&f28=&f29=&f43=&f44=&f30=EUR&f26={size}&f27={size}&f41=1&f31=&f32=&f38=%E3%F0%E0%E4+%D1%EE%F4%E8%FF&f42=&f39=&f40=&fe3=&fe4=&f45=&f46=&f51=&f52=&f33=&f34=&f35=&f36=&f37=&fe2=1";
var response = await client.PostAsync(
"https://www.imot.bg/pcgi/imot.cgi",
new StringContent(formData, Encoding.UTF8, "application/x-www-form-urlencoded"));
var firstPageUrl = response.Headers.Location;
for (var page = 1; page <= 26; page++)
{
var pageUrl = firstPageUrl.ToString().Replace("&f1=1", $"&f1={page}");
var pageHtml = await GetHtml(pageUrl);
var pageDocument = await parser.ParseDocumentAsync(pageHtml);
var listItems = pageDocument.QuerySelectorAll("a.photoLink").Where(
x => x.Attributes["href"]?.Value?.Contains("pcgi/imot.cgi?act=5&adv=") == true).ToList();
if (!properties.Any())
{
break;
}
foreach (var listItem in listItems)
{
var url = "https:" + listItem.Attributes["href"].Value;
var html = await GetHtml(url);
var document = await parser.ParseDocumentAsync(html);
var district = html.GetStringBetween(
"<span style=\"font-size:16px; font-weight:bold;\">",
"</span>")?.Trim();
if (district?.Contains("<br>") == true)
{
var indexOfBr = district.IndexOf("<br>", StringComparison.InvariantCulture);
district = district.Substring(0, indexOfBr).Trim();
}
var floorInfoString = html.GetStringBetween(
"Етаж:</td><td width=100 style=\"padding-top:3px;\"><b>",
"</b></td>");
var floorMatch = floorsRegex.Match(floorInfoString);
var typeAndInfoString = html.GetStringBetween(
"Вид строителство:</td><td width=100 style=\"padding-top:3px;\"><b>",
"</b></td>");
var typeAndInfoMatch = typeAndInfoRegex.Match(typeAndInfoString);
var property = new RawProperty
{
Url = url,
Size = size,
District = district,
Type =
html.GetStringBetween(
"<span style=\"font-size:18px; font-weight:bold;\">",
"</span>")?.Trim(),
Floor =
floorMatch.Success ? floorMatch.Groups["floor"].Value.ToInteger() : 0,
TotalFloors =
floorMatch.Success ? floorMatch.Groups["all"].Value.ToInteger() : 0,
Price =
document.QuerySelector("span#cena")?.TextContent
?.Replace(" EUR", string.Empty)?.ToInteger() ?? 0,
Year = typeAndInfoMatch.Success && typeAndInfoMatch.Groups["year"].Success
? typeAndInfoMatch.Groups["year"].Value.ToInteger()
: 0,
BuildingType =
typeAndInfoMatch.Success && typeAndInfoMatch.Groups["type"].Success
? typeAndInfoMatch.Groups["type"].Value
: null,
};
properties.Add(property);
}
Console.Write($"{page}({properties.Count}), ");
}
Console.WriteLine($" => Total: {properties.Count}");
}
return(properties);
async Task <string> GetHtml(string pageUrl)
{
var pageResponse = await client.GetAsync(pageUrl);
var byteContent = await pageResponse.Content.ReadAsByteArrayAsync();
var html = Encoding.GetEncoding("windows-1251").GetString(byteContent);
return(html);
}
}
internal string RawPropertyToPropertyString(RawProperty propertyDefinition)
{
string propertyName = RawDatabase.textBlock.GetString(propertyDefinition.NameOffset);
string fieldName = $"_{propertyName}";
bool isArray = propertyDefinition.ConversionType != ConversionType.Attribute;
// conflicting types
if (propertyName == "base")
{
propertyName = "@base";
}
if (propertyName == "params")
{
propertyName = "@params";
}
if (propertyName == "fixed")
{
propertyName = "@fixed";
}
if (propertyName == "default")
{
propertyName = "@default";
}
if (propertyName == "string")
{
propertyName = "@string";
}
if (propertyName == "switch")
{
propertyName = "@switch";
}
string typeName = RawPropertyToTypeString(propertyDefinition);
StringBuilder stringBuilder = new StringBuilder();
switch (propertyDefinition.DataType)
{
case DataType.Boolean:
case DataType.SByte:
case DataType.Int16:
case DataType.Int32:
case DataType.Int64:
case DataType.Byte:
case DataType.UInt16:
case DataType.UInt32:
case DataType.UInt64:
case DataType.String:
case DataType.Single:
case DataType.Double:
case DataType.Locale:
case DataType.Guid:
case DataType.Reference:
case DataType.Enum:
case DataType.Class:
case DataType.StrongPointer:
case DataType.WeakPointer:
break;
default:
throw new NotImplementedException();
}
switch (propertyDefinition.ConversionType)
{
case ConversionType.Attribute:
break;
case ConversionType.ComplexArray:
typeName = $"DataCoreCollection<{typeName}, IConversionComplexArray>";
break;
case ConversionType.SimpleArray:
typeName = $"DataCoreCollection<{typeName}, IConversionSimpleArray>";
break;
case ConversionType.ClassArray:
typeName = $"DataCoreCollection<{typeName}, IConversionClassArray>";
break;
}
#if !DEBUG
stringBuilder.AppendLine("[DebuggerBrowsable(DebuggerBrowsableState.Never)]");
#endif
stringBuilder.AppendLine($"{typeName} {fieldName};");
stringBuilder.AppendLine($"public {typeName} {propertyName} {{ get {{ return {fieldName}; }} set {{ SetProperty(ref {fieldName}, value); }} }}");
var result = stringBuilder.ToString();
return(result);
}
private void Populate()
{
if (Table == null || BusyPopulating)
{
return;
}
try
{
BusyPopulating = true;
Slyce.Common.Utility.SuspendPainting(this);
bool hasMultiSchemas = Table.Database.GetSchemas().Count() > 1;
//HideAllFloatingToolstrips();
Cursor = Cursors.WaitCursor;
Font boldFont = new Font(Font, FontStyle.Bold);
Font subTextFont = new Font(Font.FontFamily, Font.Size - 0.6F, FontStyle.Regular);
MainShape = new RawTable(shapeCanvas1, Table.Name, boldFont, CategoryFont, PropertyFont, Table);
if (hasMultiSchemas)
{
MainShape.SubText = string.Format(" ({0}) ", Table.Schema);
}
MainShape.FontSubText = subTextFont;
MainShape.Icon = Table.IsView ? ViewImage : TableImage;
//MainShape.Enter += new EventHandler(centreShape_Enter);
//MainShape.Leave += new EventHandler(centreShape_Leave);
MainShape.MouseClick += new MouseEventHandler(centreShape_MouseClick);
CategoryColumns.Parent = MainShape;
CategoryColumns.Properties.Clear();
MainShape.Categories.Add(CategoryColumns);
if (Table.Columns.Count > 0)
{
foreach (var column in Table.Columns)
{
RawProperty prop = new RawProperty(column.Name, column);
if (column.InPrimaryKey)
{
prop.ImageColor = Color.OrangeRed;
prop.ImageType = RawProperty.ImageTypes.Key;
}
prop.Click += new MouseEventHandler(Column_Click);
prop.DoubleClick += new MouseEventHandler(Column_DoubleClick);
CategoryColumns.Properties.Add(prop);
}
}
List <RawShape> bottomLevelShapes = new List <RawShape>();
List <RawShape> rightAlignedShapes = new List <RawShape>();
Font boldUnderlineFont = new Font(Font, FontStyle.Bold | FontStyle.Underline);
//TheEntity.Discriminator.RootGrouping.
#region Mapped Entities
if (ShowMappedEntities)
{
foreach (EntityImpl entity in MappedEntities.OrderBy(e => e.Name))
{
RawEntity entityShape = new RawEntity(shapeCanvas1, entity.Name, boldFont, entity);
CustomLineCap startCap = LineCaps.None; //.SolidArrow;
CustomLineCap endCap = LineCaps.None;
entityShape.OriginatingLineStyle.LineStyle = DashStyle.Solid;
entityShape.Cursor = Cursors.Hand;
//entityShape.OriginatingLineStyle.MiddleText = GetDiscriminatorText(entityShape);
entityShape.OriginatingLineStyle.StartCap = startCap;
entityShape.OriginatingLineStyle.EndCap = endCap;
//entityShape.BackColor1 = Color.Blue;
//entityShape.BackColor2 = Color.DarkBlue;
//entityShape.RoundedCorners = false;
entityShape.Tag = entity;
entityShape.Icon = EntityImage;
string discriminatorText = GetDiscriminatorText(entity);
entityShape.OriginatingLineStyle.MiddleText = discriminatorText;
if (string.IsNullOrEmpty(discriminatorText))
{
entityShape.OriginatingLineStyle.MiddleImage = FilterImageDisabled;
entityShape.OriginatingLineStyle.MiddleImageFocused = FilterImageDisabled;
}
else
{
entityShape.OriginatingLineStyle.MiddleImage = FilterImage;
entityShape.OriginatingLineStyle.MiddleImageFocused = FilterImageFocused;
}
entityShape.OriginatingLineStyle.MouseClick += new MouseEventHandler(OriginatingLineStyle_MouseClick);
entityShape.MouseClick += new MouseEventHandler(entityShape_MouseClick);
entityShape.OriginatingLineStyle.MiddleImageClick += new MouseEventHandler(MapLine_MiddleImageClick);
entityShape.OriginatingLineStyle.ForeColor = Color.White;
entityShape.OriginatingLineStyle.DataObject = entity;
entityShape.OriginatingLineStyle.ShowMiddleTextOnlyWhenFocused = true;
//entityShape.Enter += new EventHandler(entityShape_Enter);
//entityShape.Leave += new EventHandler(entityShape_Leave);
entityShape.MouseDoubleClick += new MouseEventHandler(entityShape_MouseDoubleClick);
//#region Setup discriminator
//if (Table.Discriminator != null && Table.Discriminator.RootGrouping != null)
//{
// ArchAngel.Providers.EntityModel.Model.DatabaseLayer.Discrimination.Condition firstCondition = Table.Discriminator.RootGrouping.Conditions.ElementAtOrDefault(0);
// if (firstCondition != null)
// {
// IColumn column = firstCondition.Column;
// ArchAngel.Providers.EntityModel.Model.DatabaseLayer.Discrimination.Operator op = firstCondition.Operator;
// string exprText = firstCondition.ExpressionValue.Value;
// }
//}
//#endregion
//tableShape.OriginatingLineStyle.MiddleImageMouseOver += new EventHandler(OriginatingLineStyle_MiddleImageMouseOver);
rightAlignedShapes.Add(entityShape);
}
#region Add empty entity
Slyce.Common.Controls.Diagramming.Shapes.RawShape emptyEntity = new Slyce.Common.Controls.Diagramming.Shapes.RawShape(shapeCanvas1, "Map entity...", boldUnderlineFont)
{
BackColor1 = Color.White,
BackColor2 = Color.White,
BorderColor = Color.Gray,
ForeColor = Color.Gray,
FocusForeColor = Color.Blue,
FocusBackColor1 = Color.WhiteSmoke,
FocusBackColor2 = Color.White,
FocusBorderColor = Color.DarkGray,
Cursor = Cursors.Hand,
Tag = null,
RoundedCorners = false,
OriginatingLineStyle = null //new LinkLine(boldFont, DashStyle.Dot, "", "", "", LineCaps.None, LineCaps.SolidArrow)
};
emptyEntity.MouseClick += new MouseEventHandler(emptyEntity_MouseClick);
//emptyTable.OriginatingLineStyle.MouseClick += new MouseEventHandler(OriginatingLineStyle_MouseClick);
rightAlignedShapes.Add(emptyEntity);
#endregion
}
#endregion
#region Relationships
if (ShowRelatedtables)
{
foreach (Relationship relationship in Table.Relationships.OrderBy(r => r.PrimaryTable == Table ? r.ForeignTable.Name : r.PrimaryTable.Name))
{
RawShape relatedTableShape;
CustomLineCap startCap;
CustomLineCap endCap;
if (relationship.PrimaryTable == Table)
{
relatedTableShape = new RawTable(shapeCanvas1, relationship.ForeignTable.Name, boldFont, relationship.ForeignTable);
if (hasMultiSchemas)
{
relatedTableShape.SubText = string.Format(" ({0}) ", relationship.ForeignTable.Schema);
}
string end1 = relationship.PrimaryKey.IsUnique ? "1" : "m";
endCap = relationship.ForeignKey.IsUnique ? LineCaps.One : LineCaps.Many;
startCap = relationship.PrimaryKey.IsUnique ? LineCaps.One : LineCaps.Many;
relatedTableShape.OriginatingLineStyle.MiddleImageClick += new MouseEventHandler(OriginatingLineStyle_MiddleImageClick2);
relatedTableShape.Icon = relationship.ForeignTable.IsView ? ViewImage : TableImage;
}
else if (relationship.ForeignTable == Table)
{
relatedTableShape = new RawTable(shapeCanvas1, relationship.PrimaryTable.Name, boldFont, relationship.PrimaryTable);
if (hasMultiSchemas)
{
relatedTableShape.SubText = string.Format(" ({0}) ", relationship.PrimaryTable.Schema);
}
string end1 = relationship.ForeignKey.IsUnique ? "1" : "m";
endCap = relationship.PrimaryKey.IsUnique ? LineCaps.One : LineCaps.Many;
startCap = relationship.ForeignKey.IsUnique ? LineCaps.One : LineCaps.Many;
relatedTableShape.OriginatingLineStyle.MiddleImageClick += new MouseEventHandler(OriginatingLineStyle_MiddleImageClick2);
relatedTableShape.Icon = relationship.PrimaryTable.IsView ? ViewImage : TableImage;
}
else
{
throw new Exception("What the...??!");
}
relatedTableShape.FontSubText = subTextFont;
if (relationship.IsUserDefined)
{
relatedTableShape.OriginatingLineStyle.LineStyle = DashStyle.Dash;
relatedTableShape.OriginatingLineStyle.LineColor = Color.Yellow;
}
else
{
relatedTableShape.OriginatingLineStyle.LineStyle = DashStyle.Solid;
}
relatedTableShape.Cursor = Cursors.Hand;
relatedTableShape.OriginatingLineStyle.StartCap = startCap;
relatedTableShape.OriginatingLineStyle.EndCap = endCap;
relatedTableShape.OriginatingLineStyle.DataObject = relationship;
relatedTableShape.OriginatingLineStyle.ForeColor = Color.White;
relatedTableShape.OriginatingLineStyle.MiddleImage = InfoImage;
relatedTableShape.MouseClick += new MouseEventHandler(referencedEntityShape_MouseClick);
relatedTableShape.MouseDoubleClick += new MouseEventHandler(referencedEntityShape_MouseDoubleClick);
bottomLevelShapes.Add(relatedTableShape);
}
// Add empty relationship
RawShape emptyTable = new RawShape(shapeCanvas1, "Add relationship...", boldUnderlineFont)
{
BackColor1 = Color.White,
BackColor2 = Color.White,
BorderColor = Color.Gray,
ForeColor = Color.Gray,
FocusForeColor = Color.Blue,
FocusBackColor1 = Color.WhiteSmoke,
FocusBackColor2 = Color.White,
FocusBorderColor = Color.DarkGray,
Cursor = Cursors.Hand,
OriginatingLineStyle = null,
Tag = null
};
emptyTable.MouseClick += new MouseEventHandler(emptyTable_MouseClick);
bottomLevelShapes.Add(emptyTable);
}
#endregion
shapeCanvas1.SwimLane1 = new ShapeCanvas.SwimLaneStyle(Color.FromArgb(79, 124, 205), Color.Black, Color.White, 0F, "Related Tables", ShapeCanvas.SwimLaneStyle.Styles.Fill);
shapeCanvas1.SwimLane3 = new ShapeCanvas.SwimLaneStyle(Color.FromArgb(79, 124, 205), Color.Black, Color.White, 180F, "Mapped Entities", ShapeCanvas.SwimLaneStyle.Styles.Fill);
shapeCanvas1.SwimLane4 = new ShapeCanvas.SwimLaneStyle(Color.FromArgb(79, 124, 205), Color.Black, Color.White, 90F, "Mapped Entities", ShapeCanvas.SwimLaneStyle.Styles.Fill);
shapeCanvas1.BackColor = this.BackColor;
//shapeCanvas1.DrawStar(centreShape, outerShapes);
//shapeCanvas1.DrawThreeLayerHorizontal(centreShape, null, outerShapes, false);
shapeCanvas1.Height = this.Height;
//shapeCanvas1.DrawThreeLayerVertical(centreShape, null, bottomLevelShapes, rightAlignedShapes);
shapeCanvas1.KeepMainShapeCentered = true;
shapeCanvas1.DrawThreeLayerHorizontal(MainShape, bottomLevelShapes, rightAlignedShapes, KeepMainShapeFull);
//shapeCanvas1.Focus();
}
finally
{
this.AutoScrollPosition = ScrollPosition;
Slyce.Common.Utility.ResumePainting(this);
Cursor = Cursors.Default;
BusyPopulating = false;
this.Focus();
shapeCanvas1.Focus();
}
}
