Request failed: HTTPConnectionPool(host='indexing_model_server', port=9000):

[akash-ak-03]

Im often getting this error. tried many ways but still facing the same issue again and again .In very rare case only its working otherwise it did't.

This issue reallyy disappointing me . kindly help me with this . I tried all slack helps from slack community but still i am facing same.

Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/urllib3/connection.py", line 199, in _new_conn
sock = connection.create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/urllib3/util/connection.py", line 85, in create_connection
raise err
File "/usr/local/lib/python3.11/site-packages/urllib3/util/connection.py", line 73, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 789, in urlopen
response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 495, in _make_request
conn.request(
File "/usr/local/lib/python3.11/site-packages/urllib3/connection.py", line 441, in request
self.endheaders()
File "/usr/local/lib/python3.11/http/client.py", line 1289, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.11/http/client.py", line 1048, in _send_output
self.send(msg)
File "/usr/local/lib/python3.11/http/client.py", line 986, in send
self.connect()
File "/usr/local/lib/python3.11/site-packages/urllib3/connection.py", line 279, in connect
self.sock = self._new_conn()
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/urllib3/connection.py", line 214, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fad78246e90>: Failed to establish a new connection: [Errno 111] Connection refused

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/requests/adapters.py", line 589, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 843, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py", line 519, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='indexing_model_server', port=9000): Max retries exceeded with url: /encoder/bi-encoder-embed (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fad78246e90>: Failed to establish a new connection: [Errno 111] Connection refused'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/app/onyx/natural_language_processing/search_nlp_models.py", line 141, in _make_model_server_request
response = final_make_request_func()
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/decorator.py", line 232, in fun
return caller(func, *(extras + args), **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/retry/api.py", line 73, in retry_decorator
return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/retry/api.py", line 33, in __retry_internal
return f()
^^^
File "/usr/local/lib/python3.11/site-packages/decorator.py", line 232, in fun
return caller(func, *(extras + args), **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/retry/api.py", line 73, in retry_decorator
return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/retry/api.py", line 33, in __retry_internal
return f()
^^^
File "/app/onyx/natural_language_processing/search_nlp_models.py", line 113, in _make_request
response = requests.post(
^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/requests/api.py", line 115, in post
return request("post", url, data=data, json=json, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/requests/adapters.py", line 622, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='indexing_model_server', port=9000): Max retries exceeded with url: /encoder/bi-encoder-embed (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fad78246e90>: Failed to establish a new connection: [Errno 111] Connection refused'))

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/app/onyx/background/indexing/run_indexing.py", line 370, in _run_indexing
index_pipeline_result = indexing_pipeline(
^^^^^^^^^^^^^^^^^^
File "/app/onyx/indexing/indexing_pipeline.py", line 168, in index_doc_batch_with_handler
index_pipeline_result = index_doc_batch(
^^^^^^^^^^^^^^^^
File "/app/onyx/utils/timing.py", line 31, in wrapped_func
result = func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/indexing/indexing_pipeline.py", line 396, in index_doc_batch
chunks_with_embeddings = embedder.embed_chunks(chunks) if chunks else []
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/utils/timing.py", line 31, in wrapped_func
result = func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/indexing/embedder.py", line 134, in embed_chunks
embeddings = self.embedding_model.encode(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/natural_language_processing/search_nlp_models.py", line 275, in encode
return self._batch_encode_texts(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/natural_language_processing/search_nlp_models.py", line 234, in _batch_encode_texts
_, batch_embeddings = process_batch(idx, text_batch)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/natural_language_processing/search_nlp_models.py", line 194, in process_batch
response = self._make_model_server_request(embed_request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/onyx/natural_language_processing/search_nlp_models.py", line 153, in _make_model_server_request
raise HTTPError(f"Request failed: {str(e)}") from e
httpx.HTTPError: Request failed: HTTPConnectionPool(host='indexing_model_server', port=9000): Max retries exceeded with url: /encoder/bi-encoder-embed (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fad78246e90>: Failed to establish a new connection: [Errno 111] Connection refused'))