Leaderboard unable to load #2523
Description
First the document need to be updated.
https://github.com/embeddings-benchmark/mteb/blob/main/docs/usage/usage.md#running-the-leaderboard-locally
Instead of using pip install mteb[gradio], users should use pip install mteb[leaderboard].
Second after installed. Run the command cause this error.
Traceback (most recent call last):
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/middleware/errors.py", line 187, in __call__
raise exc
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/middleware/errors.py", line 165, in __call__
await self.app(scope, receive, _send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/route_utils.py", line 789, in __call__
await self.app(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 714, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 734, in app
await route.handle(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/fastapi/routing.py", line 214, in run_endpoint_function
return await run_in_threadpool(dependant.call, **values)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/concurrency.py", line 37, in run_in_threadpool
return await anyio.to_thread.run_sync(func)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2470, in run_sync_in_worker_thread
return await future
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 967, in run
result = context.run(func, *args)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/routes.py", line 584, in main
gradio_api_info = api_info(request)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/routes.py", line 615, in api_info
api_info = utils.safe_deepcopy(app.get_blocks().get_api_info())
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/blocks.py", line 3048, in get_api_info
python_type = client_utils.json_schema_to_python_type(info)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 931, in json_schema_to_python_type
type_ = _json_schema_to_python_type(schema, schema.get("$defs"))
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 985, in _json_schema_to_python_type
des = [
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 986, in <listcomp>
f"{n}: {_json_schema_to_python_type(v, defs)}{get_desc(v)}"
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 993, in _json_schema_to_python_type
f"str, {_json_schema_to_python_type(schema['additionalProperties'], defs)}"
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 939, in _json_schema_to_python_type
type_ = get_type(schema)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 898, in get_type
if "const" in schema:
TypeError: argument of type 'bool' is not iterable
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/middleware/errors.py", line 187, in __call__
raise exc
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/middleware/errors.py", line 165, in __call__
await self.app(scope, receive, _send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/route_utils.py", line 789, in __call__
await self.app(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 714, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 734, in app
await route.handle(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/fastapi/routing.py", line 214, in run_endpoint_function
return await run_in_threadpool(dependant.call, **values)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/starlette/concurrency.py", line 37, in run_in_threadpool
return await anyio.to_thread.run_sync(func)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2470, in run_sync_in_worker_thread
return await future
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 967, in run
result = context.run(func, *args)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/routes.py", line 584, in main
gradio_api_info = api_info(request)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/routes.py", line 615, in api_info
api_info = utils.safe_deepcopy(app.get_blocks().get_api_info())
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/blocks.py", line 3048, in get_api_info
python_type = client_utils.json_schema_to_python_type(info)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 931, in json_schema_to_python_type
type_ = _json_schema_to_python_type(schema, schema.get("$defs"))
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 985, in _json_schema_to_python_type
des = [
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 986, in <listcomp>
f"{n}: {_json_schema_to_python_type(v, defs)}{get_desc(v)}"
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 993, in _json_schema_to_python_type
f"str, {_json_schema_to_python_type(schema['additionalProperties'], defs)}"
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 939, in _json_schema_to_python_type
type_ = get_type(schema)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio_client/utils.py", line 898, in get_type
if "const" in schema:
TypeError: argument of type 'bool' is not iterable
Traceback (most recent call last):
File "/home/seraphli/.pyenv/versions/3.10.10/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/seraphli/.pyenv/versions/3.10.10/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/mteb/leaderboard/app.py", line 890, in <module>
app.launch(server_name="0.0.0.0", server_port=7860)
File "/home/seraphli/.pyenv/versions/mteb/lib/python3.10/site-packages/gradio/blocks.py", line 2676, in launch
raise ValueError(
ValueError: When localhost is not accessible, a shareable link must be created. Please set share=True or check your proxy settings to allow access to localhost.
which is related to this issue.
gradio-app/gradio#10662
So you need to run pip install pydantic==2.10.6. [pyproject.toml](https://github.com/embeddings-benchmark/mteb/blob/main/pyproject.toml) does not fix the pydantic version, which cause this problem.