public int computeItemSimilarities(int degreeOfParallelism, int maxDurationInHours, SimilarItemsWriter writer) { ExecutorService executorService = Executors.newFixedThreadPool(degreeOfParallelism + 1); Output output = null; try { writer.open(); DataModel dataModel = getRecommender().getDataModel(); BlockingQueue<long[]> itemsIDsInBatches = queueItemIDsInBatches(dataModel, batchSize); BlockingQueue<List<SimilarItems>> results = new LinkedBlockingQueue<List<SimilarItems>>(); AtomicInteger numActiveWorkers = new AtomicInteger(degreeOfParallelism); for (int n = 0; n < degreeOfParallelism; n++) { executorService.execute(new SimilarItemsWorker(n, itemsIDsInBatches, results, numActiveWorkers)); } output = new Output(results, writer, numActiveWorkers); executorService.execute(output); } catch (Exception e) { throw new IOException(e); } finally { executorService.shutdown(); try { bool succeeded = executorService.awaitTermination(maxDurationInHours, TimeUnit.HOURS); if (!succeeded) { throw new RuntimeException("Unable to complete the computation in " + maxDurationInHours + " hours!"); } } catch (InterruptedException e) { throw new RuntimeException(e); } Closeables.close(writer, false); } return output.getNumSimilaritiesProcessed(); }
private static BlockingQueue<long[]> queueItemIDsInBatches(DataModel dataModel, int batchSize) { longPrimitiveIterator itemIDs = dataModel.getItemIDs(); int numItems = dataModel.getNumItems(); BlockingQueue<long[]> itemIDBatches = new LinkedBlockingQueue<long[]>((numItems / batchSize) + 1); long[] batch = new long[batchSize]; int pos = 0; while (itemIDs.hasNext()) { if (pos == batchSize) { itemIDBatches.add(batch.clone()); pos = 0; } batch[pos] = itemIDs.nextlong(); pos++; } int nonQueuedItemIDs = batchSize - pos; if (nonQueuedItemIDs > 0) { long[] lastBatch = new long[nonQueuedItemIDs]; Array.Copy(batch, 0, lastBatch, 0, nonQueuedItemIDs); itemIDBatches.add(lastBatch); } log.info("Queued {} items in {} batches", numItems, itemIDBatches.Count); return itemIDBatches; }