from sage.misc.package import  list_packages
from sage.features.all import all_features
from sage.env import SAGE_SRC
from os import environ, chdir, path
from subprocess import check_output, CalledProcessError
import re


pwd = environ['PWD']

spkgs = list_packages(local=True)

sl = set([k for k, v in spkgs.items() if v.type == 'standard'])
ol = set([k for k, v in spkgs.items() if v.type == 'optional'])
el = set([k for k, v in spkgs.items() if v.type == 'experimental'])

af_list = list(all_features())
af_spkg = set([f.spkg for f in af_list if f.spkg is not None])
af_name = set([f.name for f in af_list])
af = af_spkg.union(af_name)
af.add('gap_packages') # manually added since no individual instance of GapPackage is in the af_list

sl_minus_af = sl.difference(af)
ol_minus_af = ol.difference(af)
el_minus_af = el.difference(af)

sage_src = path.join(SAGE_SRC, 'sage')
chdir(sage_src)

def test_doctest_tag(spkg_name):
    cmd = 'git grep \\#[\\ ]*optional.*%s | grep -v "doctest/" | grep -v "tests/cmdline"' % spkg_name
    try:
        res = check_output(cmd, shell=True).decode()
        if re.match(r'^[^\n]*#[\s]+optional[^\n]*[\s]%s[\s,\n]' % spkg_name, res) is None:
            return False
        return True
    except CalledProcessError:
        pass
    return False

missing_standard_features = sorted([f for f in sl_minus_af if test_doctest_tag(f)])
missing_optional_features = sorted([f for f in ol_minus_af if test_doctest_tag(f)])
missing_experimental_features = sorted([f for f in el_minus_af if test_doctest_tag(f)])

chdir(pwd)

print('missing_standard_features', missing_standard_features)
print('missing_optional_features', missing_optional_features)
print('missing_experimental_features', missing_experimental_features)