From c88b9c3add1d276b42380f6f158744ada1995c84 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 26 Jul 2024 13:12:38 -0400 Subject: [PATCH] fix: changes max_queue to 0 The term fetchers were not retrieving all of the terms from solr because the thread queue was overloaded. The number of threads is bounded for these processes and it's important that all of the terms are collected, so this value is set to 0 which means infinity. Also changes ThreadPoolExecutor to FixedThreadPool in authority_browse term_fetcher so that they match. --- lib/authority_browse/term_fetcher.rb | 14 +++++++------- lib/call_number_browse/term_fetcher.rb | 5 ++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/authority_browse/term_fetcher.rb b/lib/authority_browse/term_fetcher.rb index f0d0d255..e24a1b59 100644 --- a/lib/authority_browse/term_fetcher.rb +++ b/lib/authority_browse/term_fetcher.rb @@ -84,14 +84,14 @@ def load_batch(batch) end end - # @retrun Concurrent::ThreadPoolExecutor + # @retrun Concurrent::FixedThreadPool def pool - @pool ||= Concurrent::ThreadPoolExecutor.new( - min_threads: @threads, - max_threads: @threads, - max_queue: 200, - fallback_policy: :caller_runs - ) + @pool ||= Concurrent::FixedThreadPool.new(@threads, + # max_queue: 0 means unlimited items in the queue. This is so we don't lose any + # work when shutting down. + max_queue: 0, + # fallback_policy is probably unnessary here but it won't hurt to set is explictly + fallback_policy: :caller_runs) end # Fetch all of the facets and load them into the :_from_biblio table diff --git a/lib/call_number_browse/term_fetcher.rb b/lib/call_number_browse/term_fetcher.rb index 12659c6d..6461bb8d 100644 --- a/lib/call_number_browse/term_fetcher.rb +++ b/lib/call_number_browse/term_fetcher.rb @@ -8,7 +8,10 @@ def initialize(page_size: 10_000, threads: 4) # @return Concurrent::FixedThreadPool def pool @pool ||= Concurrent::FixedThreadPool.new(@threads, - max_queue: 200, + # max_queue: 0 means unlimited items in the queue. This is so we don't lose any + # work when shutting down. + max_queue: 0, + # fallback_policy is probably unnessary here but it won't hurt to set is explictly fallback_policy: :caller_runs) end