[Core] ray raises a "Failed to unpickle serialized exception" error when an OpenAI Authentication Error is raised in task #43428
Description
What happened + What you expected to happen
When I make a call to the openai SDK with an invalid API Key, I expect to get back an Authentication error.
i.e. if I run this script
import openai
from openai import AuthenticationError
def call_openai_and_error_out():
client = openai.OpenAI(
base_url="https://api.endpoints.anyscale.com/v1",
api_key="test",
)
try:
client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are a chatbot."},
{"role": "user", "content": "What is the capital of France?"},
],
)
except AuthenticationError as e:
print("Errored as expected given API key is invalid.")
raise e
call_openai_and_error_out()I get back this exception as expected:
Errored as expected given API key is invalid.
Traceback (most recent call last):
File "/Users/marwan/Repos/deep_dives/ray-data/bug-discovery/exception-handling.py", line 24, in <module>
call_openai_and_error_out()
File "/Users/marwan/Repos/deep_dives/ray-data/bug-discovery/exception-handling.py", line 21, in call_openai_and_error_out
raise e
File "/Users/marwan/Repos/deep_dives/ray-data/bug-discovery/exception-handling.py", line 12, in call_openai_and_error_out
client.chat.completions.create(
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/openai/_utils/_utils.py", line 275, in wrapper
return func(*args, **kwargs)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/openai/resources/chat/completions.py", line 663, in create
return self._post(
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/openai/_base_client.py", line 1200, in post
return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/openai/_base_client.py", line 889, in request
return self._request(
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/openai/_base_client.py", line 980, in _request
raise self._make_status_error_from_response(err.response) from None
openai.AuthenticationError: Error code: 401 - {'generated_text': None, 'tool_calls': None, 'embedding_outputs': None, 'logprobs': None, 'num_input_tokens': None, 'num_input_tokens_batch': None, 'num_generated_tokens': None, 'num_generated_tokens_batch': None, 'preprocessing_time': None, 'generation_time': None, 'timestamp': 1708966483.861019, 'finish_reason': None, 'error': {'message': 'Your token is invalid. Please try again with a valid access token. (Request ID: 4e431ce3-4894-4c15-9859-60a7186dc0c3)', 'internal_message': "rayllm.backend.server.openai_compat.openai_exception.OpenAIHTTPException: (401, 'Your token is invalid. Please try again with a valid access token.', 'Unauthorized') (Request ID: 4e431ce3-4894-4c15-9859-60a7186dc0c3)", 'code': 401, 'type': 'OpenAIHTTPException', 'param': {}}, 'num_total_tokens': 0, 'num_total_tokens_batch': 0, 'total_time': None}
However, if I make the call_openai_and_error_out a remote function, and I attempt to execute a ray task, I now get back a "Failed to unpickle serialized exception" error obfuscating the openai.AuthenticationError.
i.e. if I run this script instead
import openai
import ray
from openai import AuthenticationError
def call_openai_and_error_out():
client = openai.OpenAI(
base_url="https://api.endpoints.anyscale.com/v1",
api_key="test",
)
try:
client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are a chatbot."},
{"role": "user", "content": "What is the capital of France?"},
],
)
except AuthenticationError as e:
print("Errored as expected given API key is invalid.")
raise e
remote_fn = ray.remote(call_openai_and_error_out)
ray.get(remote_fn.remote())I get back a Failed to unpickle serialized exception
2024-02-26 08:56:42,354 INFO worker.py:1724 -- Started a local Ray instance.
(call_openai_and_error_out pid=84736) Errored as expected given API key is invalid.
2024-02-26 08:56:45,005 ERROR serialization.py:406 -- Failed to unpickle serialized exception
Traceback (most recent call last):
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/exceptions.py", line 46, in from_ray_exception
return pickle.loads(ray_exception.serialized_exception)
TypeError: APIStatusError.__init__() missing 2 required keyword-only arguments: 'response' and 'body'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/serialization.py", line 404, in deserialize_objects
obj = self._deserialize_object(data, metadata, object_ref)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/serialization.py", line 293, in _deserialize_object
return RayError.from_bytes(obj)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/exceptions.py", line 40, in from_bytes
return RayError.from_ray_exception(ray_exception)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/exceptions.py", line 49, in from_ray_exception
raise RuntimeError(msg) from e
RuntimeError: Failed to unpickle serialized exception
Traceback (most recent call last):
File "/Users/marwan/Repos/deep_dives/ray-data/bug-discovery/exception-handling.py", line 26, in <module>
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(*args, **kwargs)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(*args, **kwargs)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/worker.py", line 2626, in get
raise value
ray.exceptions.RaySystemError: System error: Failed to unpickle serialized exception
traceback: Traceback (most recent call last):
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/exceptions.py", line 46, in from_ray_exception
return pickle.loads(ray_exception.serialized_exception)
TypeError: APIStatusError.__init__() missing 2 required keyword-only arguments: 'response' and 'body'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/serialization.py", line 404, in deserialize_objects
obj = self._deserialize_object(data, metadata, object_ref)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/_private/serialization.py", line 293, in _deserialize_object
return RayError.from_bytes(obj)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/exceptions.py", line 40, in from_bytes
return RayError.from_ray_exception(ray_exception)
File "/Users/marwan/.pyenv/versions/ray-data-bug-discovery-py310/lib/python3.10/site-packages/ray/exceptions.py", line 49, in from_ray_exception
raise RuntimeError(msg) from e
RuntimeError: Failed to unpickle serialized exceptionVersions / Dependencies
python version: 3.10.11 (main, Dec 12 2023, 16:25:48) [Clang 15.0.0 (clang-1500.0.40.1)]
ray version: 2.9.3
openai version: 1.12.0
Reproduction script
See the above snippet to reproduce
Issue Severity
Low: It annoys or frustrates me.