public void CacheChunk(Chunk chunk)
{
if (!string.IsNullOrWhiteSpace(chunk.Id) && _chunkCache.Any(x => x.Id == chunk.Id))
{
throw new InvalidOperationException(string.Format("Attempted to re-cache cached chunk {0}", chunk.Id));
}
if (_chunkCache.Contains(chunk))
{
throw new InvalidOperationException("Attempted to re-cache unsaved chunk");
}
_chunkCache.Add(chunk);
}
public bool Update(Feed feed)
{
// Now, we've already added some articles, maybe.
// These have been added to the head chunk, which might be oversized.
// Let's say they haven't been added to feed.Articles.
var headChunk = feed.GetHeadChunk(_db);
_logger.DebugFormat("incoming feed has {0} saved chunks and {1} cached chunks already", feed.ChunkIds.Count, feed.CachedChunkCount);
while (headChunk.Articles.Count > MaxArticlesPerChunk)
{
_logger.InfoFormat("reshuffling chunks for feed {0}", feed);
var oldHead = headChunk;
headChunk = new Chunk();
feed.SetHeadChunk(headChunk);
headChunk.Articles = oldHead.Articles.OrderBy(x => x.PublishDate).Skip(MaxArticlesPerChunk).ToList();
oldHead.Articles = oldHead.Articles.OrderBy(x => x.PublishDate).Take(MaxArticlesPerChunk).ToList();
_logger.DebugFormat("old head has {0} articles; new has {1}", oldHead.Articles.Count, headChunk.Articles.Count);
}
feed.Save(_db);
// Okay, let's rebuild feed.Articles.
// This is loading way too much data...
feed.Articles.Clear();
foreach (var id in feed.ChunkIds)
{
var chunk = feed.GetChunk(id, _db);
if (chunk == null) {
_logger.WarnFormat("feed {0} missing chunk {1}", feed.Id, id);
continue;
}
feed.Articles.AddRange(chunk.Articles);
}
feed.Articles = feed.Articles.OrderByDescending(x => x.PublishDate).Take(MaxArticlesPerChunk).Reverse().ToList();
//feed.Articles = feed.ChunkIds.Select(x => feed.GetChunk(x, _db)).Where(x => x != null).SelectMany(x => x.Articles).OrderByDescending(x => x.PublishDate).Take(MaxArticlesPerChunk).Reverse().ToList();
feed.Save(_db);
_logger.DebugFormat("outgoing feed has {0} saved chunks", feed.ChunkIds.Count);
return true;
}
public Chunk GetHeadChunk(Mongo db)
{
if (_head != null)
{
return _head;
}
if (string.IsNullOrEmpty(HeadChunkId))
{
HeadChunkId = ChunkIds.LastOrDefault();
if (string.IsNullOrEmpty(HeadChunkId))
{
_head = new Chunk { FeedId = Id };
CacheChunk(_head);
return _head;
}
}
_head = GetChunk(HeadChunkId, db);
if (_head == null)
{
_head = new Chunk { FeedId = Id };
CacheChunk(_head);
}
return _head;
}
public void SetHeadChunk(Chunk value)
{
_head = value;
// TODO set Start to a sensible value
HeadChunkId = value.Id;
CacheChunk(value);
}
public void Save(Mongo db)
{
if (string.IsNullOrEmpty(Id))
{
// Ensure we have an id that we can set for our chunks.
db.Feeds.Save(this);
}
// shouldn't happen...
if (_head != null && !_chunkCache.Contains(_head))
{
_chunkCache.Add(_head);
}
foreach (var chunk in _chunkCache.Where(x => !x.Articles.Any()))
{
Console.WriteLine("removing chunk {0} because it has {1} articles", chunk.Id, chunk.Articles.Count);
ChunkIds.Remove(chunk.Id);
}
foreach (var chunk in _chunkCache.Where(x => x.Articles.Any()).OrderBy(x => x.Articles.First().PublishDate))
{
Console.WriteLine("saving chunk {0} with {1} articles", chunk.Id, chunk.Articles.Count);
chunk.FeedId = this.Id;
chunk.Save(db);
if (!ChunkIds.Contains(chunk.Id))
{
ChunkIds.Add(chunk.Id);
}
}
if (_head != null)
{
if (string.IsNullOrWhiteSpace(_head.Id))
{
_head = null;
HeadChunkId = ChunkIds.LastOrDefault();
}
else if (HeadChunkId != _head.Id)
{
HeadChunkId = _head.Id;
}
}
db.Feeds.Save(this);
}