Skip to content

Use platformdirs to improve database path handling #40152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 1, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ dependencies = [
'typing_extensions >= 4.4.0; python_version<"3.11"',
'ipython >=8.9.0',
'pexpect >=4.8.0',
'platformdirs',
'sphinx >=5.2, <9',
'networkx >=3.1',
'scipy >=1.11',
Expand Down
56 changes: 42 additions & 14 deletions src/sage/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
....: cmd += f"s1 = samefile(SAGE_ROOT, '{SAGE_ROOT}');"
sage: cmd += f"s2 = samefile(SAGE_LOCAL, '{SAGE_LOCAL}');"
sage: cmd += "print(s1 and s2);"
sage: out = check_output([sys.executable, "-c", cmd], env=env).decode().strip() # long time

Check failure on line 22 in src/sage/env.py

View workflow job for this annotation

GitHub Actions / Conda (ubuntu, Python 3.12, new)

Failed example:

Failed example:: Exception raised: Traceback (most recent call last): File "/usr/share/miniconda/envs/sage-dev/lib/python3.12/site-packages/sage/doctest/forker.py", line 730, in _run self.compile_and_execute(example, compiler, test.globs) File "/usr/share/miniconda/envs/sage-dev/lib/python3.12/site-packages/sage/doctest/forker.py", line 1154, in compile_and_execute exec(compiled, globs) File "<doctest sage.env[8]>", line 1, in <module> out = check_output([sys.executable, "-c", cmd], env=env).decode().strip() # long time ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/share/miniconda/envs/sage-dev/lib/python3.12/subprocess.py", line 468, in check_output return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/share/miniconda/envs/sage-dev/lib/python3.12/subprocess.py", line 573, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command '['/usr/share/miniconda/envs/sage-dev/bin/python3', '-c', "from sage.all import SAGE_ROOT, SAGE_LOCAL;from os.path import samefile;s1 = samefile(SAGE_ROOT, '/home/runner/work/sage/sage');s2 = samefile(SAGE_LOCAL, '/usr/share/miniconda/envs/sage-dev');print(s1 and s2);"]' returned non-zero exit status 1.
sage: out == "True" # long time

Check failure on line 23 in src/sage/env.py

View workflow job for this annotation

GitHub Actions / Conda (ubuntu, Python 3.12, new)

Failed example:

Failed example:: Exception raised: Traceback (most recent call last): File "/usr/share/miniconda/envs/sage-dev/lib/python3.12/site-packages/sage/doctest/forker.py", line 730, in _run self.compile_and_execute(example, compiler, test.globs) File "/usr/share/miniconda/envs/sage-dev/lib/python3.12/site-packages/sage/doctest/forker.py", line 1154, in compile_and_execute exec(compiled, globs) File "<doctest sage.env[9]>", line 1, in <module> out == "True" # long time ^^^ NameError: name 'out' is not defined
True

AUTHORS:
Expand All @@ -39,22 +39,22 @@
# https://www.gnu.org/licenses/
# ****************************************************************************

from typing import Optional
import sage
import platform
import os
import socket
import subprocess
import sys
import sysconfig
from . import version
import subprocess
from typing import Optional

from platformdirs import site_data_dir, user_data_dir

from sage import version

# All variables set by var() appear in this SAGE_ENV dict
SAGE_ENV = dict()


def join(*args):
def join(*args) -> str | None:
"""
Join paths like ``os.path.join`` except that the result is ``None``
if any of the components is ``None``.
Expand Down Expand Up @@ -215,13 +215,9 @@
SAGE_ARCHFLAGS = var("SAGE_ARCHFLAGS", "unset")
SAGE_PKG_CONFIG_PATH = var("SAGE_PKG_CONFIG_PATH")

# colon-separated search path for databases.
SAGE_DATA_PATH = var("SAGE_DATA_PATH",
os.pathsep.join(filter(None, [
join(DOT_SAGE, "db"),
join(SAGE_SHARE, "sagemath"),
SAGE_SHARE,
])))
# colon-separated search path for databases
# should not be used directly; instead use sage_data_paths
SAGE_DATA_PATH = var("SAGE_DATA_PATH")

# database directories, the default is to search in SAGE_DATA_PATH
CREMONA_LARGE_DATA_DIR = var("CREMONA_LARGE_DATA_DIR")
Expand Down Expand Up @@ -414,9 +410,10 @@
....: ''')
435
"""
import pkgconfig
import itertools

import pkgconfig

if required_modules is None:
required_modules = default_required_modules

Expand Down Expand Up @@ -515,3 +512,34 @@
aliases["OPENMP_CXXFLAGS"] = OPENMP_CXXFLAGS.split()

return aliases


def sage_data_paths(name: str | None) -> set[str]:
r"""
Search paths for general data files.

If specified, the subdirectory ``name`` is appended to the
directories. Otherwise, the directories are returned as is.

EXAMPLES::

sage: from sage.env import sage_data_paths
sage: sage_data_paths("cremona")
{'.../cremona'}
"""
if not SAGE_DATA_PATH:
paths = {
join(DOT_SAGE, "db"),
join(SAGE_SHARE, "sagemath"),
SAGE_SHARE,
}
paths.add(user_data_dir("sagemath"))
paths.add(user_data_dir())
paths.add(site_data_dir("sagemath"))
paths.add(site_data_dir())
else:
paths = {path for path in SAGE_DATA_PATH.split(os.pathsep)}

if name is None:
return {path for path in paths if path}
return {os.path.join(path, name) for path in paths if path}
147 changes: 77 additions & 70 deletions src/sage/features/databases.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,27 +17,8 @@
# https://www.gnu.org/licenses/
# *****************************************************************************

import os

from . import StaticFile, PythonModule
from sage.env import SAGE_DATA_PATH


def sage_data_path(data_name):
r"""
Search path for database ``data_name``.

EXAMPLES::

sage: from sage.features.databases import sage_data_path
sage: sage_data_path("cremona")
['.../cremona']
"""
if not SAGE_DATA_PATH:
return []

return [os.path.join(p, data_name)
for p in SAGE_DATA_PATH.split(os.pathsep)]
from sage.env import sage_data_paths
from sage.features import PythonModule, StaticFile


class DatabaseCremona(StaticFile):
Expand All @@ -58,32 +39,39 @@ class DatabaseCremona(StaticFile):
sage: DatabaseCremona().is_present() # optional - database_cremona_ellcurve
FeatureTestResult('database_cremona_ellcurve', True)
"""
def __init__(self, name='cremona', spkg='database_cremona_ellcurve', type='optional'):

def __init__(
self, name="cremona", spkg="database_cremona_ellcurve", type="optional"
):
r"""
TESTS::

sage: from sage.features.databases import DatabaseCremona
sage: isinstance(DatabaseCremona(), DatabaseCremona)
True
"""
from sage.env import CREMONA_MINI_DATA_DIR, CREMONA_LARGE_DATA_DIR
from sage.env import CREMONA_LARGE_DATA_DIR, CREMONA_MINI_DATA_DIR

CREMONA_DATA_DIRS = set([CREMONA_MINI_DATA_DIR, CREMONA_LARGE_DATA_DIR])
CREMONA_DATA_DIRS.discard(None)
search_path = CREMONA_DATA_DIRS or sage_data_path("cremona")
search_path = CREMONA_DATA_DIRS or sage_data_paths("cremona")

spkg = "database_cremona_ellcurve"
spkg_type = "optional"
if name == 'cremona_mini':
if name == "cremona_mini":
spkg = "elliptic_curves"
spkg_type = "standard"

StaticFile.__init__(self, f"database_{name}_ellcurve",
filename=f"{name}.db",
search_path=search_path,
spkg=spkg,
type=spkg_type,
url='https://github.com/JohnCremona/ecdata',
description="Cremona's database of elliptic curves")
StaticFile.__init__(
self,
f"database_{name}_ellcurve",
filename=f"{name}.db",
search_path=search_path,
spkg=spkg,
type=spkg_type,
url="https://github.com/JohnCremona/ecdata",
description="Cremona's database of elliptic curves",
)


class DatabaseEllcurves(StaticFile):
Expand All @@ -97,6 +85,7 @@ class DatabaseEllcurves(StaticFile):
sage: bool(DatabaseEllcurves().is_present()) # optional - database_ellcurves
True
"""

def __init__(self):
r"""
TESTS::
Expand All @@ -106,14 +95,18 @@ def __init__(self):
True
"""
from sage.env import ELLCURVE_DATA_DIR
search_path = ELLCURVE_DATA_DIR or sage_data_path("ellcurves")

StaticFile.__init__(self, "database_ellcurves",
filename='rank0',
search_path=search_path,
spkg='elliptic_curves',
type='standard',
description="William Stein's database of interesting curve")
search_path = ELLCURVE_DATA_DIR or sage_data_paths("ellcurves")

StaticFile.__init__(
self,
"database_ellcurves",
filename="rank0",
search_path=search_path,
spkg="elliptic_curves",
type="standard",
description="William Stein's database of interesting curve",
)


class DatabaseGraphs(StaticFile):
Expand All @@ -127,6 +120,7 @@ class DatabaseGraphs(StaticFile):
sage: bool(DatabaseGraphs().is_present()) # optional - database_graphs
True
"""

def __init__(self):
r"""
TESTS::
Expand All @@ -136,14 +130,18 @@ def __init__(self):
True
"""
from sage.env import GRAPHS_DATA_DIR
search_path = GRAPHS_DATA_DIR or sage_data_path("graphs")

StaticFile.__init__(self, "database_graphs",
filename='graphs.db',
search_path=search_path,
spkg='graphs',
type='standard',
description="A database of graphs")
search_path = GRAPHS_DATA_DIR or sage_data_paths("graphs")

StaticFile.__init__(
self,
"database_graphs",
filename="graphs.db",
search_path=search_path,
spkg="graphs",
type="standard",
description="A database of graphs",
)


class DatabaseJones(StaticFile):
Expand All @@ -157,6 +155,7 @@ class DatabaseJones(StaticFile):
sage: bool(DatabaseJones().is_present()) # optional - database_jones_numfield
True
"""

def __init__(self):
r"""
TESTS::
Expand All @@ -165,11 +164,14 @@ def __init__(self):
sage: isinstance(DatabaseJones(), DatabaseJones)
True
"""
StaticFile.__init__(self, "database_jones_numfield",
filename='jones.sobj',
search_path=sage_data_path("jones"),
spkg='database_jones_numfield',
description="John Jones's tables of number fields")
StaticFile.__init__(
self,
"database_jones_numfield",
filename="jones.sobj",
search_path=sage_data_paths("jones"),
spkg="database_jones_numfield",
description="John Jones's tables of number fields",
)


class DatabaseKnotInfo(PythonModule):
Expand All @@ -187,6 +189,7 @@ class DatabaseKnotInfo(PythonModule):
sage: DatabaseKnotInfo().is_present() # optional - database_knotinfo
FeatureTestResult('database_knotinfo', True)
"""

def __init__(self):
r"""
TESTS::
Expand All @@ -195,7 +198,7 @@ def __init__(self):
sage: isinstance(DatabaseKnotInfo(), DatabaseKnotInfo)
True
"""
PythonModule.__init__(self, 'database_knotinfo', spkg='database_knotinfo')
PythonModule.__init__(self, "database_knotinfo", spkg="database_knotinfo")


class DatabaseMatroids(PythonModule):
Expand All @@ -213,6 +216,7 @@ class DatabaseMatroids(PythonModule):

[Mat2012]_
"""

def __init__(self):
r"""
TESTS::
Expand All @@ -221,7 +225,7 @@ def __init__(self):
sage: isinstance(DatabaseMatroids(), DatabaseMatroids)
True
"""
PythonModule.__init__(self, 'matroid_database', spkg='matroid_database')
PythonModule.__init__(self, "matroid_database", spkg="matroid_database")


class DatabaseCubicHecke(PythonModule):
Expand All @@ -239,6 +243,7 @@ class DatabaseCubicHecke(PythonModule):
sage: DatabaseCubicHecke().is_present() # optional - database_cubic_hecke
FeatureTestResult('database_cubic_hecke', True)
"""

def __init__(self):
r"""
TESTS::
Expand All @@ -247,7 +252,7 @@ def __init__(self):
sage: isinstance(DatabaseCubicHecke(), DatabaseCubicHecke)
True
"""
PythonModule.__init__(self, 'database_cubic_hecke', spkg='database_cubic_hecke')
PythonModule.__init__(self, "database_cubic_hecke", spkg="database_cubic_hecke")


class DatabaseReflexivePolytopes(StaticFile):
Expand All @@ -264,7 +269,8 @@ class DatabaseReflexivePolytopes(StaticFile):
sage: bool(DatabaseReflexivePolytopes('polytopes_db_4d').is_present()) # optional - polytopes_db_4d
True
"""
def __init__(self, name='polytopes_db'):

def __init__(self, name="polytopes_db"):
"""
TESTS::

Expand All @@ -277,26 +283,27 @@ def __init__(self, name='polytopes_db'):
'Hodge4d'
"""
from sage.env import POLYTOPE_DATA_DIR
search_path = POLYTOPE_DATA_DIR or sage_data_path("reflexive_polytopes")

search_path = POLYTOPE_DATA_DIR or sage_data_paths("reflexive_polytopes")

dirname = "Full3d"
if name == "polytopes_db_4d":
dirname = "Hodge4d"

StaticFile.__init__(self, name,
filename=dirname,
search_path=search_path)
StaticFile.__init__(self, name, filename=dirname, search_path=search_path)


def all_features():
return [PythonModule('conway_polynomials', spkg='conway_polynomials', type='standard'),
DatabaseCremona(),
DatabaseCremona('cremona_mini', type='standard'),
DatabaseEllcurves(),
DatabaseGraphs(),
DatabaseJones(),
DatabaseKnotInfo(),
DatabaseMatroids(),
DatabaseCubicHecke(),
DatabaseReflexivePolytopes(),
DatabaseReflexivePolytopes('polytopes_db_4d')]
return [
PythonModule("conway_polynomials", spkg="conway_polynomials", type="standard"),
DatabaseCremona(),
DatabaseCremona("cremona_mini", type="standard"),
DatabaseEllcurves(),
DatabaseGraphs(),
DatabaseJones(),
DatabaseKnotInfo(),
DatabaseMatroids(),
DatabaseCubicHecke(),
DatabaseReflexivePolytopes(),
DatabaseReflexivePolytopes("polytopes_db_4d"),
]
Loading