public void DecodeSegment(Segment segment)
{
segment.status = Segment.Status.YDECODING;
lock (segments)
{
segments.Enqueue(segment);
}
}
public void addSegment(ref Segment segment)
{
segment.status = Segment.Status.WRITECACHE;
lock (newSegments)
{
newSegments.Add(segment);
}
}
private string getFilename(ref Segment segment)
{
// Create output dictionary if necessary
Directory.CreateDirectory(Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\");
if (segment.yname != null && segment.yname != "")
{
return Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\" + segment.yname;
}
else
{
return Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\" + segment.parent.filename;
}
}
private void writeSegment(Segment seg, WriteStream ws)
{
ws.stream.Write(seg.data, 0, seg.data.Length);
seg.parent.byteprogress += seg.data.Length;
seg.data = null;
seg.status = Segment.Status.COMPLETE;
seg.parent.saveprogress = seg.id;
ws.lastuse = DateTime.Now.Millisecond;
}
// Updates cache
private void Update()
{
while (keepRunning)
{
List<Segment> toremove = new List<Segment>();
Segment[] segs = new Segment[newSegments.Count];
lock (newSegments)
{
newSegments.CopyTo(segs);
newSegments.Clear();
}
for (int i = 0; i < segs.Length; i++)
{
processSegment(segs[i]);
}
foreach (Segment seg in segments)
{
if (seg.id - 1 == seg.parent.saveprogress)
{
saveSegment(seg);
cacheSize -= (ulong)seg.bytes;
toremove.Add(seg);
}
}
foreach (Segment seg in toremove)
{
segments.Remove(seg);
}
// Empty out streams thats passed the timeout limit..
foreach (WriteStream ws in streams)
{
if (ws.lastuse + STREAMTIMEOUT * 1000 < DateTime.Now.Millisecond)
{
ws.stream.Close();
}
}
Thread.Sleep(1000); // Sleep 1sec before updating again
}
}
private void saveSegment(Segment seg)
{
System.Console.WriteLine("Saving seg: " + seg.id);
bool saved = false;
// Check if stream is in cache
List<WriteStream> toremove = new List<WriteStream>();
List<WriteStream>.Enumerator enumer = streams.GetEnumerator();
if(enumer.MoveNext()) {
do
{
WriteStream ws = (WriteStream)enumer.Current;
if (ws.filejob.Equals(seg.parent))
{
// Copy from a tempsaved segment
if (seg.tempsaved)
{
FileStream stream = new FileStream(Properties.Settings.Default.tempFolder + "\\" + seg.tempname, FileMode.Open, FileAccess.Read);
byte[] buffer = new byte[seg.bytes];
stream.Read(buffer, 0, seg.bytes); // May end up blocking if seg.bytes is larger than actual file
seg.data = buffer;
stream.Close();
stream.Dispose();
}
writeSegment(seg, ws);
saved = true;
// Is filejob complete?
if (seg.id == seg.parent.yparts)
{
// Close stream
ws.stream.Flush();
ws.stream.Close();
ws.stream.Dispose();
// Remove from collection
toremove.Add(ws);
seg.parent.complete = true;
}
break;
}
}
while (enumer.MoveNext());
}
foreach (WriteStream ws in toremove)
{
streams.Remove(ws);
}
// Else open a new stream
if (!saved)
{
// Open new stream..
WriteStream ws = new WriteStream();
ws.stream = new FileStream(seg.parent.outputfilename, FileMode.Append, FileAccess.Write);
ws.filejob = seg.parent;
writeSegment(seg, ws);
// Was this the last part?
if (seg.id == seg.parent.yparts || seg.parent.yparts == 0)
{
// Close stream..
ws.stream.Flush();
ws.stream.Close();
ws.stream.Dispose();
}
else
{
streams.Add(ws);
}
}
}
// Processes new segment
private void processSegment(Segment segment)
{
// is the cache waiting for this part?
if (segment.id == 1)
{
Logging.Instance.Log("(Cache) Recieved first segment");
// First part in a filejob..
segment.parent.outputfilename = getFilename(ref segment);
saveSegment(segment);
}
else if (segment.id - 1 == segment.parent.saveprogress)
{
// Append to file
Logging.Instance.Log("(Cache) Recieved awaited segment");
saveSegment(segment);
}
else if (cacheSize + (ulong)segment.bytes > (ulong)MAXCACHE * 1024 * 1024)
{
Logging.Instance.Log("Cache too big, saving to disk");
segment.status = Segment.Status.TEMPCACHED;
segment.tempname = "temp" + new Random().Next(9999999);
FileStream stream = new FileStream(Properties.Settings.Default.tempFolder + "\\" + segment.tempname, FileMode.Create, FileAccess.Write);
stream.Write(segment.data, 0, segment.bytes);
segment.data = null;
stream.Flush();
stream.Close();
stream.Dispose();
segments.Add(segment);
segment.tempsaved = true;
}
else
{
// memorycache
Logging.Instance.Log("(Cache) Saving segment to cache");
segments.Add(segment);
cacheSize += (ulong)segment.bytes;
}
}
public byte[] RecieveSegment(Segment segment)
{
try
{
// Request body...
Write("BODY "+ segment.addr +'\n');
Assert(Response(), "222");
NetworkStream stream = GetStream();
stream.ReadTimeout = 10000;
byte[] buffer = new byte[segment.bytes];
int read = 0;
bool done = false;
int chunk;
while (!done && (chunk = stream.Read(buffer, read, buffer.Length - read)) > 0 && keepAlive)
{
read += chunk;
segment.parent.parent.ProgressKB += (ulong)chunk;
segment.progress += chunk;
lock (this)
{
speed += (uint)chunk;
}
if (read == buffer.Length)
{
byte[] newBuffer = new byte[buffer.Length * 2];
Array.Copy(buffer, newBuffer, buffer.Length);
buffer = newBuffer;
}
mainGui.UpdateSingleThreadInfo(this);
// Looks for .\r\n (End of message)
if (read > 2 && buffer[read - 3] == '.' && buffer[read - 2] == '\r' && buffer[read - 1] == '\n')
{
// Check that it is in the beginning of a new line
if (read < 4 || buffer[read - 5] == '\r' && buffer[read - 4] == '\n')
{
break;
}
}
}
if (!keepAlive)
{
throw new Exception("KeepAlive is false"); ;
}
byte[] returnarray = new byte[read - 3];
Array.Copy(buffer, returnarray, read - 3);
segment.bytes = read;
Logging.Instance.Log("(NNTPConnection(" + id + ")) SEGMENT Complete: id=" + segment.id + " bytes=" + read);
return returnarray;
}
catch (IOException ex)
{
throw;
}
}
private void ProcessSegment(Segment segment)
{
int expectedlenght = 0; // TODO: Check accuracy of this value
MemoryStream ms = new MemoryStream(segment.data);
string header = ReadLine(ms);
while(header.Equals(""))
header = ReadLine(ms);
int ysize = 0; // TODO: Check accuracy of this value
if (header.Contains("=ybegin ") && header.Contains("line=") && header.Contains("size=") && header.Contains("name="))
{
try
{
segment.isYenc = true;
ysize = int.Parse(getValue(header, "size"));
segment.yname = getValue(header, "name");
segment.yname = EnsureGoodName(segment.yname);
segment.parent.filename = segment.yname;
if (header.Contains("part="))
{
segment.YPart = int.Parse(getValue(header, "part"));
if (segment.YPart > segment.parent.yparts)
segment.parent.yparts = segment.YPart;
string partheader = ReadLine(ms);
if (partheader.Contains("=ypart ") && partheader.Contains("begin=") && partheader.Contains("end="))
{
int partBegin = int.Parse(getValue(partheader, "begin"));
int partEnd = int.Parse(getValue(partheader, "end"));
expectedlenght = partEnd - partBegin;
}
}
}
catch (Exception ex)
{
}
}
if (!segment.isYenc)
{
segment.bytes = segment.data.Length;
WriteCache.Instance.addSegment(ref segment);
return;
}
else
{
// Start decoding data...
byte[] buffer = new byte[1];
byte[] output = new byte[segment.data.Length];
bool cr = false, dot = false, finished = false, decode = true, nl = false, bleh2 = false; // flags
int bufpos = 0, lastnl = 0;
// Decode byte-for-byte
while (ms.Read(buffer, 0, 1) > 0 && !finished)
{
if (cr && buffer[0] == '\n')
{
// Discard \r\n by going back one byte and discarding current byte
bufpos--;
lastnl = bufpos;
decode = false;
}
else if (dot && buffer[0] == '.' && bufpos == lastnl+1)
{
// Discard current (2nd) dot
//string s = "";
//for (int i = 10; i > 0; i--)
//{
// s += (char)output[bufpos - 1 - i];
//}
//System.Console.WriteLine("LastNL: {0}, last10: {1}", lastnl, s);
decode = false;
}
// Reset flags
cr = false;
dot = false;
// Set flags
switch ((int)buffer[0])
{
case '=':
ms.Read(buffer, 0, 1);
if (buffer[0] == 'y')
{
string yend = ReadLine(ms);
if (segment.YPart != 0)
segment.crc = getValue(yend, "pcrc32");
else
segment.crc = getValue(yend, "crc32");
finished = true;
break;
} else {
for (int i = 0; i < special.Length; i++)
{
if(buffer[0] == special[i]+64)
buffer[0] = (byte)(char)special[i];
}
}
break;
case '\r':
cr = true;
break;
case '.':
if(decode)
dot = true;
break;
}
if (decode && !finished)
{
// Decode byte
buffer[0] -= 42 %256;
output[bufpos] = buffer[0];
bufpos++;
}
decode = true;
}
byte[] data = new byte[bufpos];
Array.Copy(output, data, bufpos);
segment.data = data;
segment.bytes = segment.data.Length;
segment.ValidateCRC();
WriteCache.Instance.addSegment(ref segment);
}
}
//// Generates queues for a filecollection object
//public static Queue genQueue(ArrayList list)
//{
// Queue queue = new Queue(list.Count);
// foreach (FileJob job in list)
// {
// job.queue = new Queue(job.segments.Count);
// foreach (Segment seg in job.segments)
// {
// job.queue.Enqueue(seg);
// }
// queue.Enqueue(job);
// }
// return queue;
//}
// Parses XML structure into file objects
private static List<FileJob> ParseXML(XmlDocument document, FileCollection filecollection)
{
XmlNodeList nodes = document.GetElementsByTagName("file");
List<FileJob> files = new List<FileJob>();
Random random = new Random();
int collectionID = random.Next();
int fileId = 0;
// Parsing for each file node
foreach (XmlElement fileNode in nodes)
{
FileJob filejob = new FileJob(ref filecollection);
filejob.filename = ""+collectionID;
XmlAttributeCollection collection = fileNode.Attributes;
// File Attributes
foreach (XmlAttribute attr in collection)
{
switch (attr.Name)
{
case "poster":
filejob.poster = attr.Value;
break;
case "date":
filejob.date = int.Parse(attr.Value);
break;
case "subject":
if (attr.Value.Contains("\""))
{
// try to get inner filename...
try
{
int first = attr.Value.IndexOf("\"") + 1;
string filename = attr.Value.Substring(first, attr.Value.Length - first);
filename = filename.Substring(0, filename.IndexOf("\""));
filejob.filename = filejob.filename + "-"+ filename.Substring(0, filename.LastIndexOf(".")+4);
}
catch (Exception ex)
{
filejob.filename = filejob.filename+"-"+fileId;
}
}
else
filejob.filename = filejob.filename + "-" + fileId;
filejob.subject = attr.Value;
break;
}
}
int maxnumber = 0;
foreach (XmlElement subNode in fileNode.ChildNodes)
{
if (subNode.Name == "segments")
{
foreach (XmlElement segmentNode in subNode)
{
Segment segment = new Segment();
segment.addr = "<"+segmentNode.InnerXml+">";
segment.addr = segment.addr.Replace("&", "&");
segment.setParent(ref filejob);
foreach (XmlAttribute attr in segmentNode.Attributes)
{
switch (attr.Name)
{
case "bytes":
segment.bytes = int.Parse(attr.Value);
filejob.size += (ulong)segment.bytes;
break;
case "number":
segment.id = int.Parse(attr.Value);
if (segment.id > maxnumber)
maxnumber = segment.id;
break;
}
}
// Check and add element to arraylist
if (segment.id != 0 && segment.bytes != 0 && segment.addr.Length > 10)
filejob.segments.Add(segment);
}
} else if (subNode.Name == "groups")
{
filejob.groups.Add(subNode.FirstChild.InnerText);
}
}
filejob.yparts = maxnumber; // Will fail if source isnt yenc
// Sort segments
filejob.segments.Sort(new SegmentComparer());
files.Add(filejob);
fileId++;
}
return files;
}
