public int computeItemSimilarities(int degreeOfParallelism, int maxDurationInHours, SimilarItemsWriter writer)
{
ExecutorService executorService = Executors.newFixedThreadPool(degreeOfParallelism + 1);
Output output = null;
try {
writer.open();
DataModel dataModel = getRecommender().getDataModel();
BlockingQueue<long[]> itemsIDsInBatches = queueItemIDsInBatches(dataModel, batchSize);
BlockingQueue<List<SimilarItems>> results = new LinkedBlockingQueue<List<SimilarItems>>();
AtomicInteger numActiveWorkers = new AtomicInteger(degreeOfParallelism);
for (int n = 0; n < degreeOfParallelism; n++) {
executorService.execute(new SimilarItemsWorker(n, itemsIDsInBatches, results, numActiveWorkers));
}
output = new Output(results, writer, numActiveWorkers);
executorService.execute(output);
} catch (Exception e) {
throw new IOException(e);
} finally {
executorService.shutdown();
try {
bool succeeded = executorService.awaitTermination(maxDurationInHours, TimeUnit.HOURS);
if (!succeeded) {
throw new RuntimeException("Unable to complete the computation in " + maxDurationInHours + " hours!");
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
Closeables.close(writer, false);
}
return output.getNumSimilaritiesProcessed();
}
private static BlockingQueue<long[]> queueItemIDsInBatches(DataModel dataModel, int batchSize) {
longPrimitiveIterator itemIDs = dataModel.getItemIDs();
int numItems = dataModel.getNumItems();
BlockingQueue<long[]> itemIDBatches = new LinkedBlockingQueue<long[]>((numItems / batchSize) + 1);
long[] batch = new long[batchSize];
int pos = 0;
while (itemIDs.hasNext()) {
if (pos == batchSize) {
itemIDBatches.add(batch.clone());
pos = 0;
}
batch[pos] = itemIDs.nextlong();
pos++;
}
int nonQueuedItemIDs = batchSize - pos;
if (nonQueuedItemIDs > 0) {
long[] lastBatch = new long[nonQueuedItemIDs];
Array.Copy(batch, 0, lastBatch, 0, nonQueuedItemIDs);
itemIDBatches.add(lastBatch);
}
log.info("Queued {} items in {} batches", numItems, itemIDBatches.Count);
return itemIDBatches;
}