/// <summary> /// When overridden in a derived type, this method will perform the actual parsing of information - from the read html to the proper object of type <typeparamref name="T"/> or a derivative. /// This method may never return null. /// </summary> /// <param name="html">The html-code that is to be converted.</param> /// <param name="request">The requested url from the http-request.</param> /// <param name="response">The response url from the http-request.</param> /// <returns>An object of type <typeparamref name="T"/> or one of it derivatives, that represents the loaded page.</returns> protected abstract T Convert(string html, URL request, URL response);
/// <summary>
/// Removes an url-address from the <see cref="HTMLBuffer{T}"/>.
/// </summary>
/// <param name="url">The address to remove.</param>
/// <returns>A boolean value indicating if the address was removed (if it pre-existed).</returns>
protected bool Remove(URL url)
{
return(itemDictionary.Remove(url));
}
/// <summary>
/// Reads a <see cref="URL"/> address using the default encoding.
/// </summary>
/// <param name="url">The <see cref="URL"/> to read.</param>
/// <returns>An object of type <typeparamref name="T"/> or one of it derivatives, that represents the loaded page.
/// This is read from buffer, if preloaded; or via the <see cref="Convert"/> method, if not.</returns>
protected T ReadURL(URL url)
{
return(ReadURL(url, this.encoding));
}
/// <summary>
/// Reads a <see cref="URL"/> address using the specified encoding.
/// </summary>
/// <param name="url">The <see cref="URL"/> to read.</param>
/// <param name="encoding">The encoding used for reading.</param>
/// <returns>An object of type <typeparamref name="T"/> or one of it derivatives, that represents the loaded page.
/// This is read from buffer, if preloaded; or via the <see cref="Convert"/> method, if not.</returns>
protected T ReadURL(URL url, Encoding encoding)
{
if (url == null)
{
throw new ArgumentNullException("url");
}
if (encoding == null)
{
throw new ArgumentNullException("encoding");
}
url = translate(url);
T item = null;
if (itemDictionary.ContainsKey(url))
{
item = itemDictionary[url];
}
if (item == null)
{
if (fileDictionary.ContainsKey(url))
{
string html;
URL responseURL;
using (FileStream fs = new FileStream(this.filepath, FileMode.Open))
fileDictionary[url].Read(fs, out html, out responseURL);
item = Convert(html, url, responseURL);
if (item == null)
{
throw new InvalidOperationException("The " + typeof(Convert).Name + " method may not return null.");
}
item.Buffer = this;
if (itemDictionary.ContainsKey(url))
{
itemDictionary[url] = item;
}
else
{
itemDictionary.Add(url, item);
}
}
else
{
URL newurl;
string html = url.GetHTML(encoding, out newurl);
newurl = translate(newurl);
item = Convert(html, url, newurl);
if (item == null)
{
throw new InvalidOperationException("The " + typeof(Convert).Name + " method may not return null.");
}
item.Buffer = this;
itemDictionary.Add(url, item);
if (url != newurl && !itemDictionary.ContainsKey(newurl))
{
T item2 = Convert(html, newurl, newurl);
if (item2 == null)
{
throw new InvalidOperationException("The " + typeof(Convert).Name + " method may not return null.");
}
item2.Buffer = this;
itemDictionary.Add(newurl, item2);
}
}
}
return(item);
}
/// <summary>
/// Writes part of the buffered data to a specified file, for later retrieval.
/// </summary>
/// <param name="filepath">The file to which the buffered data should be written. The file is overridden.</param>
/// <param name="shouldSave">A function that, from bufferlevels, determines which pages should be stored.</param>
public void Save(string filepath, Func <int, bool> shouldSave)
{
List <URL> requestList = new List <URL>();
List <URL> responseList = new List <URL>();
List <string> htmlList = new List <string>();
List <int> bufferList = new List <int>();
foreach (var var in itemDictionary)
{
URL request = var.Key;
URL response = null;
string html = null;
int buffer;
if (var.Value != null)
{
response = var.Value.ResponseURL;
html = var.Value.HTML;
buffer = var.Value.BufferLevel;
}
else if (loadedFromFile)
{
buffer = fileDictionary[var.Key].BufferLevel;
if (shouldSave(buffer))
{
using (FileStream fs = new FileStream(filepath, FileMode.Open))
fileDictionary[var.Key].Read(fs, out html, out response);
}
}
else
{
throw new InvalidOperationException("There can be no null-values in the buffer.");
}
if (shouldSave(buffer))
{
requestList.Add(request);
responseList.Add(response);
htmlList.Add(html);
bufferList.Add(buffer);
}
}
using (FileStream fs = new FileStream(filepath, FileMode.Create))
using (BinaryWriter writer = new BinaryWriter(fs))
for (int i = 0; i < requestList.Count; i++)
{
byte[] req = Encoding.Unicode.GetBytes(requestList[i].Address);
byte[] res = Encoding.Unicode.GetBytes(responseList[i].Address);
byte[] html = Encoding.Unicode.GetBytes(htmlList[i]);
writer.Write((byte)0);
writer.Write(bufferList[i]);
writer.Write(req.Length);
writer.Write(req);
writer.Write(res.Length);
writer.Write(res);
writer.Write(html.Length);
writer.Write(html);
}
}
