From d447ee63a079362a89f4030e64826e3949aacb25 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Tue, 10 Dec 2024 21:10:39 -0800 Subject: [PATCH 1/7] generate conda env yml and pip requirements files automatically from pyproject.toml using script --- docs/.readthedocs.yaml | 2 +- docs/requirements-rtd.txt | 6 + docs/requirements.txt | 3 - docs/source/conf.py | 2 +- environments/environments.json | 51 ++++++ environments/make-env-files.py | 167 ++++++++++++++++++ environments/requirements.txt | 70 +++----- environments/tests/env-ci.yml | 53 +++--- environments/tests/env-test-minimum-deps.yml | 51 +++--- .../tests/requirements-test-latest-deps.txt | 38 ++-- pyproject.toml | 2 + 11 files changed, 319 insertions(+), 126 deletions(-) create mode 100644 docs/requirements-rtd.txt delete mode 100644 docs/requirements.txt create mode 100644 environments/environments.json create mode 100644 environments/make-env-files.py diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml index cefab32d0..0a10f343b 100644 --- a/docs/.readthedocs.yaml +++ b/docs/.readthedocs.yaml @@ -9,7 +9,7 @@ formats: all python: install: - - requirements: docs/requirements.txt + - requirements: docs/requirements-rtd.txt sphinx: configuration: docs/source/conf.py diff --git a/docs/requirements-rtd.txt b/docs/requirements-rtd.txt new file mode 100644 index 000000000..42fd6389c --- /dev/null +++ b/docs/requirements-rtd.txt @@ -0,0 +1,6 @@ +# Do not edit this file. It is automatically generated by the script +# environments/make-env-files.py using the environment definition data +# in environments/environments.json and the requirements in pyproject.toml. +furo +sphinx-autodoc-typehints +sphinx>=7 diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 532390375..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -furo -sphinx == 7.* # same value as needs_sphinx in /docs/source/conf.py -sphinx-autodoc-typehints diff --git a/docs/source/conf.py b/docs/source/conf.py index 4b9adb840..78e5776b0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -63,7 +63,7 @@ html_static_path: list[str] = [] html_theme = "furo" language = "en" -needs_sphinx = "7" # same value as pinned in /docs/requirements.txt +needs_sphinx = "7" # match version from pyproject.toml optional-dependencies root_doc = "index" source_suffix = ".rst" templates_path: list[str] = [] diff --git a/environments/environments.json b/environments/environments.json new file mode 100644 index 000000000..7897e5082 --- /dev/null +++ b/environments/environments.json @@ -0,0 +1,51 @@ +{ + "env-test-minimum-deps": { + "filepath": "./environments/tests/env-test-minimum-deps.yml", + "force_pin": true, + "which_optionals": [ + "entropy", + "neighbors", + "raster", + "visualization", + "tests" + ] + }, + "env-ci": { + "filepath": "./environments/tests/env-ci.yml" + }, + "requirements-test-latest-deps": { + "filepath": "./environments/tests/requirements-test-latest-deps.txt", + "needs_python": false + }, + "requirements-rtd": { + "filepath": "./docs/requirements-rtd.txt", + "needs_python": false, + "needs_dependencies": false, + "which_optionals": [ + "docs" + ] + }, + "requirements-env": { + "filepath": "./environments/requirements.txt", + "needs_python": false, + "extras": [ + "bottleneck", + "cartopy", + "conda-smithy", + "folium", + "hatch", + "jupyterlab", + "nbdime", + "nbqa", + "numexpr", + "pillow", + "pip", + "pysal>24", + "python-igraph", + "seaborn", + "statsmodels", + "twine", + "validate-pyproject" + ] + } +} diff --git a/environments/make-env-files.py b/environments/make-env-files.py new file mode 100644 index 000000000..cca3a1922 --- /dev/null +++ b/environments/make-env-files.py @@ -0,0 +1,167 @@ +# noqa: INP001 +"""Make conda env.yml and pip requirements.txt files from environments.json data.""" + +from __future__ import annotations + +import argparse +import itertools +import json +from pathlib import Path + +import tomllib +from packaging.requirements import Requirement + +# path to package's pyproject and the config json file +pyproject_path = "./pyproject.toml" +environments_config_path = "./environments/environments.json" + +# what channels to specify in conda env yml files +CHANNELS = ["conda-forge"] + +# if any envs in config json are missing these keys, use these default values +ENV_DEFAULTS = { + "filepath": "./requirements-TEST.txt", + "needs_python": True, + "needs_dependencies": True, + "needs_optionals": True, + "which_optionals": None, + "force_pin": False, + "extras": None, +} + +HEADER = ( + "# Do not edit this file. It is automatically generated by the script\n" + "# environments/make-env-files.py using the environment definition data\n" + "# in environments/environments.json and the requirements in pyproject.toml.\n" +) + + +def extract_optional_deps(which: list[str] | None = None) -> list[Requirement]: + """ + Extract a list of the optional dependencies/versions from pyproject.toml. + + Parameters + ---------- + which + Which optional dependencies to extract. If None, extract them all. + + Returns + ------- + optional_deps + """ + opts = pyproject["project"]["optional-dependencies"] + opts = [v for k, v in opts.items() if k in which] if which is not None else opts.values() + return list({Requirement(o) for o in itertools.chain.from_iterable(opts)}) + + +def make_requirement( + requirement: Requirement, + force_pin: bool = False, # noqa: FBT001,FBT002 + is_conda: bool = True, # noqa: FBT001,FBT002 +) -> str: + """ + Make a requirement specification string. + + The string result comprises the requirement's name and its specifier(s). + + Parameters + ---------- + requirement + A requirement object + force_pin + If True, pin requirement to version rather than using existing + specifier. Allows you to convert minimum versions to pinned versions. + is_conda + If True and if `force_pin` is True, format the requirement string to + end with ".*" for conda environment file pinning format compatibility. + + Returns + ------- + requirement_str + """ + specifiers = list(requirement.specifier) + if force_pin and len(specifiers) == 1: + spec = f"{requirement.name}=={specifiers[0].version}" + if is_conda and not spec.endswith(".*"): + spec += ".*" + return spec + return str(requirement) + + +def make_file(env_name: str) -> None: + """ + Write a conda environment yaml file or pip requirements.txt file. + + Parameters + ---------- + env_name + An enviroment name among the keys of environments.json. + + Returns + ------- + None + """ + # fill in any missing configurations with default values + env = envs[env_name] + env = {k: env.get(k, v) for k, v in ENV_DEFAULTS.items()} + + # it's a conda env file if it ends with ".yml", otherwise it's a pip + # requirements.txt file + is_conda = env["filepath"].endswith(".yml") + + # determine which dependencies to add based on the configuration + depends_on = [] + if env["needs_python"]: + python_dep = Requirement(f"python{pyproject['project']['requires-python']}") + depends_on.append(python_dep) + if env["needs_dependencies"]: + dependencies = [Requirement(d) for d in pyproject["project"]["dependencies"]] + depends_on.extend(dependencies) + if env["needs_optionals"]: + optionals = extract_optional_deps(which=env["which_optionals"]) + depends_on.extend(optionals) + + # make the list of requirements + requirements = sorted( + make_requirement(dep, force_pin=env["force_pin"], is_conda=is_conda) for dep in depends_on + ) + + # add any extra requirements if provided in the configuration + if env["extras"] is not None: + requirements = sorted(requirements + env["extras"]) + + # write the conda env yml or pip requirements.txt file to disk + with Path(env["filepath"]).open("w") as f: + if is_conda: + data = {"name": env_name, "channels": CHANNELS, "dependencies": requirements} + text = "" + for k, v in data.items(): + if isinstance(v, list): + text += k + ":\n- " + "\n- ".join(v) + "\n" + elif isinstance(v, str): + text += k + ": " + v + "\n" + f.writelines(HEADER + text) + else: + f.writelines(HEADER + "\n".join(requirements) + "\n") + + print(f"Wrote {len(requirements)} requirements to {env['filepath']!r}") # noqa: T201 + + +if __name__ == "__main__": + # load the pyproject.toml and the environments.json config files + pyproject = tomllib.loads(Path(pyproject_path).read_text()) + with Path(environments_config_path).open() as f: + envs = json.load(f) + + # parse any command-line arguments passed by the user + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("-n", dest="env_name", type=str) + args = arg_parser.parse_args() + + if args.env_name is not None: + # if user passed -n command line argument, generate only that file + make_file(args.env_name) + else: + # otherwise, make all environment files + for env_name in envs: + make_file(env_name) diff --git a/environments/requirements.txt b/environments/requirements.txt index 40f53fc82..ef9892c51 100644 --- a/environments/requirements.txt +++ b/environments/requirements.txt @@ -1,53 +1,39 @@ -# required dependencies -geopandas -networkx -numpy -pandas -requests -shapely - -# optional dependencies -matplotlib -rasterio -rio-vrt -scipy -scikit-learn - -# helpful extras +# Do not edit this file. It is automatically generated by the script +# environments/make-env-files.py using the environment definition data +# in environments/environments.json and the requirements in pyproject.toml. bottleneck cartopy +conda-smithy folium -jupyterlab -numexpr -pillow -pysal -python-igraph -seaborn -statsmodels - -# docs furo -sphinx -sphinx-autodoc-typehints - -# packaging -conda-smithy +geopandas>=1.0 hatch -pip -twine -validate-pyproject - -# typing -mypy -pandas-stubs -typeguard -types-requests - -# linting/testing +jupyterlab lxml +matplotlib>=3.5 nbdime nbqa +networkx>=2.5 +numexpr +numpy>=1.22 +pandas>=1.4 +pillow +pip pre-commit +pysal>24 pytest pytest-cov -ruff +python-igraph +rasterio>=1.3 +requests>=2.27 +rio-vrt>=0.3 +scikit-learn>=0.23 +scipy>=1.5 +seaborn +shapely>=2.0 +sphinx-autodoc-typehints +sphinx>=7 +statsmodels +twine +typeguard +validate-pyproject diff --git a/environments/tests/env-ci.yml b/environments/tests/env-ci.yml index b90b48d20..c37f1db17 100644 --- a/environments/tests/env-ci.yml +++ b/environments/tests/env-ci.yml @@ -1,32 +1,27 @@ +# Do not edit this file. It is automatically generated by the script +# environments/make-env-files.py using the environment definition data +# in environments/environments.json and the requirements in pyproject.toml. name: env-ci - channels: - - conda-forge - +- conda-forge dependencies: - # requirements - - geopandas - - networkx - - numpy - - pandas - - requests - - shapely - - # extras - - matplotlib - - rasterio - - rio-vrt - - scikit-learn - - scipy - - # linting/testing - - lxml - - pre-commit - - pytest - - pytest-cov - - typeguard - - # docs - - furo - - sphinx=7 - - sphinx-autodoc-typehints +- furo +- geopandas>=1.0 +- lxml +- matplotlib>=3.5 +- networkx>=2.5 +- numpy>=1.22 +- pandas>=1.4 +- pre-commit +- pytest +- pytest-cov +- python>=3.9 +- rasterio>=1.3 +- requests>=2.27 +- rio-vrt>=0.3 +- scikit-learn>=0.23 +- scipy>=1.5 +- shapely>=2.0 +- sphinx-autodoc-typehints +- sphinx>=7 +- typeguard diff --git a/environments/tests/env-test-minimum-deps.yml b/environments/tests/env-test-minimum-deps.yml index 6ab64ab4e..162db638a 100644 --- a/environments/tests/env-test-minimum-deps.yml +++ b/environments/tests/env-test-minimum-deps.yml @@ -1,31 +1,24 @@ -# Pin dependencies to minimum required minor versions. Allows you to create a -# conda env to run the test suite against the oldest supported versions of -# Python and the package's required dependencies. -name: env-test-minimal - +# Do not edit this file. It is automatically generated by the script +# environments/make-env-files.py using the environment definition data +# in environments/environments.json and the requirements in pyproject.toml. +name: env-test-minimum-deps channels: - - conda-forge - +- conda-forge dependencies: - # pin minimum supported python version from pyproject.toml - - python=3.9 - - # pin required dependencies' min versions from pyproject.toml - - geopandas=1.0 - - networkx=2.5 - - numpy=1.22 - - pandas=1.4 - - requests=2.27 - - shapely=2.0 - - # pin optional dependencies' min versions from pyproject.toml - - matplotlib=3.5 - - rasterio=1.3 - - rio-vrt=0.3 - - scikit-learn=0.23 - - scipy=1.5 - - # testing - - lxml - - pytest - - typeguard +- geopandas==1.0.* +- lxml +- matplotlib==3.5.* +- networkx==2.5.* +- numpy==1.22.* +- pandas==1.4.* +- pre-commit +- pytest +- pytest-cov +- python==3.9.* +- rasterio==1.3.* +- requests==2.27.* +- rio-vrt==0.3.* +- scikit-learn==0.23.* +- scipy==1.5.* +- shapely==2.0.* +- typeguard diff --git a/environments/tests/requirements-test-latest-deps.txt b/environments/tests/requirements-test-latest-deps.txt index 4f059fcb5..957dd3c20 100644 --- a/environments/tests/requirements-test-latest-deps.txt +++ b/environments/tests/requirements-test-latest-deps.txt @@ -1,26 +1,22 @@ -# requirements -geopandas -networkx -numpy -pandas -requests -shapely - -# extras -matplotlib -rasterio -rio-vrt -scikit-learn -scipy - -# linting/testing +# Do not edit this file. It is automatically generated by the script +# environments/make-env-files.py using the environment definition data +# in environments/environments.json and the requirements in pyproject.toml. +furo +geopandas>=1.0 lxml +matplotlib>=3.5 +networkx>=2.5 +numpy>=1.22 +pandas>=1.4 pre-commit pytest pytest-cov -typeguard - -# docs -furo -sphinx == 7.* +rasterio>=1.3 +requests>=2.27 +rio-vrt>=0.3 +scikit-learn>=0.23 +scipy>=1.5 +shapely>=2.0 sphinx-autodoc-typehints +sphinx>=7 +typeguard diff --git a/pyproject.toml b/pyproject.toml index c58d3f5f7..88bad7f22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,11 @@ readme = "README.md" requires-python = ">=3.9" # match classifiers above and ruff/mypy versions below [project.optional-dependencies] +docs = ["furo", "sphinx >=7", "sphinx-autodoc-typehints"] entropy = ["scipy>=1.5"] neighbors = ["scikit-learn>=0.23", "scipy>=1.5"] raster = ["rasterio>=1.3", "rio-vrt>=0.3"] +tests = ["lxml", "pre-commit", "pytest", "pytest-cov", "typeguard"] visualization = ["matplotlib>=3.5"] [project.urls] From bd18b26e008aab5ca8cc682934a0f64b86de007f Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Tue, 10 Dec 2024 21:24:37 -0800 Subject: [PATCH 2/7] update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 88bad7f22..6defe7a4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,10 +39,10 @@ license = { text = "MIT License" } maintainers = [{ name = "OSMnx contributors" }] name = "osmnx" readme = "README.md" -requires-python = ">=3.9" # match classifiers above and ruff/mypy versions below +requires-python = ">=3.9" # match classifiers above and mypy version below [project.optional-dependencies] -docs = ["furo", "sphinx >=7", "sphinx-autodoc-typehints"] +docs = ["furo", "sphinx>=7", "sphinx-autodoc-typehints"] entropy = ["scipy>=1.5"] neighbors = ["scikit-learn>=0.23", "scipy>=1.5"] raster = ["rasterio>=1.3", "rio-vrt>=0.3"] From 609722c3b6f12e7741ef3afa0612090a24dcc6a1 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 12 Dec 2024 13:53:12 -0800 Subject: [PATCH 3/7] update bash options --- .github/workflows/build-publish-pypi.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/test-docs-linkcheck.yml | 2 +- .github/workflows/test-latest-deps.yml | 2 +- .github/workflows/test-minimum-deps.yml | 2 +- environments/docker/build-image.sh | 2 +- environments/unix-create-env.sh | 11 ++++++----- tests/lint_test.sh | 2 +- tests/prune.sh | 2 +- 9 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-publish-pypi.yml b/.github/workflows/build-publish-pypi.yml index 7f02bff77..6fbd3ccb2 100644 --- a/.github/workflows/build-publish-pypi.yml +++ b/.github/workflows/build-publish-pypi.yml @@ -23,7 +23,7 @@ jobs: defaults: run: - shell: bash -elo pipefail {0} + shell: bash -euo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0168198e9..4e40ebe25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: defaults: run: - shell: bash -elo pipefail {0} + shell: bash -euo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/test-docs-linkcheck.yml b/.github/workflows/test-docs-linkcheck.yml index 7450bc180..99f63f14e 100644 --- a/.github/workflows/test-docs-linkcheck.yml +++ b/.github/workflows/test-docs-linkcheck.yml @@ -15,7 +15,7 @@ jobs: defaults: run: - shell: bash -elo pipefail {0} + shell: bash -euo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/test-latest-deps.yml b/.github/workflows/test-latest-deps.yml index f64e51ef2..f1b49c3d8 100644 --- a/.github/workflows/test-latest-deps.yml +++ b/.github/workflows/test-latest-deps.yml @@ -20,7 +20,7 @@ jobs: defaults: run: - shell: bash -elo pipefail {0} + shell: bash -euo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/test-minimum-deps.yml b/.github/workflows/test-minimum-deps.yml index 1375064cb..b5dfbcef5 100644 --- a/.github/workflows/test-minimum-deps.yml +++ b/.github/workflows/test-minimum-deps.yml @@ -20,7 +20,7 @@ jobs: defaults: run: - shell: bash -elo pipefail {0} + shell: bash -euo pipefail {0} steps: - name: Checkout repo diff --git a/environments/docker/build-image.sh b/environments/docker/build-image.sh index 3c1988b6b..753ee5404 100755 --- a/environments/docker/build-image.sh +++ b/environments/docker/build-image.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail echo "Run this script from the repository root." docker login docker buildx build --no-cache --pull --push --platform=linux/amd64,linux/arm64 -f ./environments/docker/Dockerfile -t gboeing/osmnx:test . diff --git a/environments/unix-create-env.sh b/environments/unix-create-env.sh index 0c90ce779..8a656ead9 100755 --- a/environments/unix-create-env.sh +++ b/environments/unix-create-env.sh @@ -1,18 +1,19 @@ #!/bin/bash -set -e +set -euo pipefail +echo "Run conda deactivate before running this script." ENV=ox ENV_PATH=$(conda info --base)/envs/$ENV PACKAGE=osmnx eval "$(conda shell.bash hook)" conda activate base -conda env remove -n $ENV --yes -mamba create -n $ENV --yes -c conda-forge --strict-channel-priority --file requirements.txt -eval "$(conda shell.bash hook)" +conda env remove --yes -n $ENV || true +conda update --yes -c conda-forge --strict-channel-priority -n base conda mamba +mamba create --yes -c conda-forge --strict-channel-priority -n $ENV --file requirements.txt conda activate $ENV python -m pip --python $ENV_PATH uninstall $PACKAGE --yes python -m pip --python $ENV_PATH install -e ../. -python -m pip --python $ENV_PATH check python -m ipykernel install --prefix $ENV_PATH --name $ENV --display-name "Python ($ENV)" conda list -n $ENV +python -m pip --python $ENV_PATH check jupyter kernelspec list ipython -c "import $PACKAGE; print('$PACKAGE version', $PACKAGE.__version__)" diff --git a/tests/lint_test.sh b/tests/lint_test.sh index 8ca3814e7..f17e0e039 100755 --- a/tests/lint_test.sh +++ b/tests/lint_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail # delete temp files and folders rm -r -f ./.coverage* ./.pytest_cache ./.temp ./dist ./docs/build ./*/__pycache__ diff --git a/tests/prune.sh b/tests/prune.sh index d0752f687..333d11b02 100755 --- a/tests/prune.sh +++ b/tests/prune.sh @@ -1,6 +1,6 @@ #!/bin/bash # prune conda, docker, and git -set -e +set -euo pipefail eval "$(conda shell.bash hook)" conda activate base conda clean --all --yes From 248525603aa38fa93645dbd7a1b39405b1f6eecb Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 12 Dec 2024 14:00:02 -0800 Subject: [PATCH 4/7] update scripts --- environments/unix-create-env.sh | 1 - environments/windows-create-env.bat | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/environments/unix-create-env.sh b/environments/unix-create-env.sh index 8a656ead9..43b6d4dc0 100755 --- a/environments/unix-create-env.sh +++ b/environments/unix-create-env.sh @@ -7,7 +7,6 @@ PACKAGE=osmnx eval "$(conda shell.bash hook)" conda activate base conda env remove --yes -n $ENV || true -conda update --yes -c conda-forge --strict-channel-priority -n base conda mamba mamba create --yes -c conda-forge --strict-channel-priority -n $ENV --file requirements.txt conda activate $ENV python -m pip --python $ENV_PATH uninstall $PACKAGE --yes diff --git a/environments/windows-create-env.bat b/environments/windows-create-env.bat index ed87f26b9..1be878fbc 100644 --- a/environments/windows-create-env.bat +++ b/environments/windows-create-env.bat @@ -4,13 +4,13 @@ SET ENV_PATH=%CONDA_ROOT%\envs\%ENV% SET PACKAGE=osmnx CALL %CONDA_ROOT%\Scripts\activate.bat && ^ conda activate base && ^ -conda env remove -n %ENV% --yes && ^ -mamba create -n %ENV% --yes -c conda-forge --strict-channel-priority --file requirements.txt && ^ +conda env remove --yes -n %ENV% && ^ +mamba create --yes -c conda-forge --strict-channel-priority -n %ENV% --file requirements.txt && ^ conda activate %ENV% && ^ python -m pip --python %ENV_PATH%\python.exe uninstall %PACKAGE% --yes && ^ python -m pip --python %ENV_PATH%\python.exe install -e ../. && ^ -python -m pip --python %ENV_PATH%\python.exe check && ^ python -m ipykernel install --prefix %ENV_PATH% --name %ENV% --display-name "Python (%ENV%)" && ^ conda list && ^ +python -m pip --python %ENV_PATH%\python.exe check && ^ jupyter kernelspec list && ^ ipython -c "import %PACKAGE%; print('%PACKAGE% version', %PACKAGE%.__version__)" From f7f701d9be5de193b796b7949757e0aa7dcd7afb Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 12 Dec 2024 14:16:52 -0800 Subject: [PATCH 5/7] move defaults out of script --- environments/environments.json | 38 +++++++++++++++++++++++----------- environments/make-env-files.py | 22 +++++--------------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/environments/environments.json b/environments/environments.json index 7897e5082..064998dfb 100644 --- a/environments/environments.json +++ b/environments/environments.json @@ -1,33 +1,47 @@ { "env-test-minimum-deps": { "filepath": "./environments/tests/env-test-minimum-deps.yml", + "needs_python": true, + "needs_dependencies": true, + "needs_optionals": true, + "which_optionals": ["entropy", "neighbors", "raster", "visualization", "tests"], "force_pin": true, - "which_optionals": [ - "entropy", - "neighbors", - "raster", - "visualization", - "tests" - ] + "extras": null }, "env-ci": { - "filepath": "./environments/tests/env-ci.yml" + "filepath": "./environments/tests/env-ci.yml", + "needs_python": true, + "needs_dependencies": true, + "needs_optionals": true, + "which_optionals": null, + "force_pin": false, + "extras": null }, "requirements-test-latest-deps": { "filepath": "./environments/tests/requirements-test-latest-deps.txt", - "needs_python": false + "needs_python": false, + "needs_dependencies": true, + "needs_optionals": true, + "which_optionals": null, + "force_pin": false, + "extras": null }, "requirements-rtd": { "filepath": "./docs/requirements-rtd.txt", "needs_python": false, "needs_dependencies": false, - "which_optionals": [ - "docs" - ] + "needs_optionals": true, + "which_optionals": ["docs"], + "force_pin": false, + "extras": null }, "requirements-env": { "filepath": "./environments/requirements.txt", "needs_python": false, + "needs_dependencies": true, + "needs_optionals": true, + "which_optionals": null, + "force_pin": false, "extras": [ "bottleneck", "cartopy", diff --git a/environments/make-env-files.py b/environments/make-env-files.py index cca3a1922..7b27983db 100644 --- a/environments/make-env-files.py +++ b/environments/make-env-files.py @@ -18,21 +18,10 @@ # what channels to specify in conda env yml files CHANNELS = ["conda-forge"] -# if any envs in config json are missing these keys, use these default values -ENV_DEFAULTS = { - "filepath": "./requirements-TEST.txt", - "needs_python": True, - "needs_dependencies": True, - "needs_optionals": True, - "which_optionals": None, - "force_pin": False, - "extras": None, -} - HEADER = ( "# Do not edit this file. It is automatically generated by the script\n" - "# environments/make-env-files.py using the environment definition data\n" - "# in environments/environments.json and the requirements in pyproject.toml.\n" + "# environments/make-env-files.py using the environment definition data in\n" + "# environments/environments.json and the requirements in pyproject.toml.\n" ) @@ -101,9 +90,7 @@ def make_file(env_name: str) -> None: ------- None """ - # fill in any missing configurations with default values env = envs[env_name] - env = {k: env.get(k, v) for k, v in ENV_DEFAULTS.items()} # it's a conda env file if it ends with ".yml", otherwise it's a pip # requirements.txt file @@ -149,8 +136,9 @@ def make_file(env_name: str) -> None: if __name__ == "__main__": # load the pyproject.toml and the environments.json config files - pyproject = tomllib.loads(Path(pyproject_path).read_text()) - with Path(environments_config_path).open() as f: + with Path(pyproject_path).open("rb") as f: + pyproject = tomllib.load(f) + with Path(environments_config_path).open("rb") as f: envs = json.load(f) # parse any command-line arguments passed by the user From f6b78d799f50f27d8953b61f5158d3137624d244 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 12 Dec 2024 14:17:01 -0800 Subject: [PATCH 6/7] update header text --- docs/requirements-rtd.txt | 4 ++-- environments/requirements.txt | 4 ++-- environments/tests/env-ci.yml | 4 ++-- environments/tests/env-test-minimum-deps.yml | 4 ++-- environments/tests/requirements-test-latest-deps.txt | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/requirements-rtd.txt b/docs/requirements-rtd.txt index 42fd6389c..c05a3595c 100644 --- a/docs/requirements-rtd.txt +++ b/docs/requirements-rtd.txt @@ -1,6 +1,6 @@ # Do not edit this file. It is automatically generated by the script -# environments/make-env-files.py using the environment definition data -# in environments/environments.json and the requirements in pyproject.toml. +# environments/make-env-files.py using the environment definition data in +# environments/environments.json and the requirements in pyproject.toml. furo sphinx-autodoc-typehints sphinx>=7 diff --git a/environments/requirements.txt b/environments/requirements.txt index ef9892c51..0802377ca 100644 --- a/environments/requirements.txt +++ b/environments/requirements.txt @@ -1,6 +1,6 @@ # Do not edit this file. It is automatically generated by the script -# environments/make-env-files.py using the environment definition data -# in environments/environments.json and the requirements in pyproject.toml. +# environments/make-env-files.py using the environment definition data in +# environments/environments.json and the requirements in pyproject.toml. bottleneck cartopy conda-smithy diff --git a/environments/tests/env-ci.yml b/environments/tests/env-ci.yml index c37f1db17..2363615ff 100644 --- a/environments/tests/env-ci.yml +++ b/environments/tests/env-ci.yml @@ -1,6 +1,6 @@ # Do not edit this file. It is automatically generated by the script -# environments/make-env-files.py using the environment definition data -# in environments/environments.json and the requirements in pyproject.toml. +# environments/make-env-files.py using the environment definition data in +# environments/environments.json and the requirements in pyproject.toml. name: env-ci channels: - conda-forge diff --git a/environments/tests/env-test-minimum-deps.yml b/environments/tests/env-test-minimum-deps.yml index 162db638a..4944aa868 100644 --- a/environments/tests/env-test-minimum-deps.yml +++ b/environments/tests/env-test-minimum-deps.yml @@ -1,6 +1,6 @@ # Do not edit this file. It is automatically generated by the script -# environments/make-env-files.py using the environment definition data -# in environments/environments.json and the requirements in pyproject.toml. +# environments/make-env-files.py using the environment definition data in +# environments/environments.json and the requirements in pyproject.toml. name: env-test-minimum-deps channels: - conda-forge diff --git a/environments/tests/requirements-test-latest-deps.txt b/environments/tests/requirements-test-latest-deps.txt index 957dd3c20..2b7c9a387 100644 --- a/environments/tests/requirements-test-latest-deps.txt +++ b/environments/tests/requirements-test-latest-deps.txt @@ -1,6 +1,6 @@ # Do not edit this file. It is automatically generated by the script -# environments/make-env-files.py using the environment definition data -# in environments/environments.json and the requirements in pyproject.toml. +# environments/make-env-files.py using the environment definition data in +# environments/environments.json and the requirements in pyproject.toml. furo geopandas>=1.0 lxml From b28811560831c7b3eee37078e27a311508018e15 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Thu, 12 Dec 2024 15:01:31 -0800 Subject: [PATCH 7/7] restore bash options --- .github/workflows/build-publish-pypi.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/test-docs-linkcheck.yml | 2 +- .github/workflows/test-latest-deps.yml | 2 +- .github/workflows/test-minimum-deps.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-publish-pypi.yml b/.github/workflows/build-publish-pypi.yml index 6fbd3ccb2..7f02bff77 100644 --- a/.github/workflows/build-publish-pypi.yml +++ b/.github/workflows/build-publish-pypi.yml @@ -23,7 +23,7 @@ jobs: defaults: run: - shell: bash -euo pipefail {0} + shell: bash -elo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e40ebe25..0168198e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: defaults: run: - shell: bash -euo pipefail {0} + shell: bash -elo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/test-docs-linkcheck.yml b/.github/workflows/test-docs-linkcheck.yml index 99f63f14e..7450bc180 100644 --- a/.github/workflows/test-docs-linkcheck.yml +++ b/.github/workflows/test-docs-linkcheck.yml @@ -15,7 +15,7 @@ jobs: defaults: run: - shell: bash -euo pipefail {0} + shell: bash -elo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/test-latest-deps.yml b/.github/workflows/test-latest-deps.yml index f1b49c3d8..f64e51ef2 100644 --- a/.github/workflows/test-latest-deps.yml +++ b/.github/workflows/test-latest-deps.yml @@ -20,7 +20,7 @@ jobs: defaults: run: - shell: bash -euo pipefail {0} + shell: bash -elo pipefail {0} steps: - name: Checkout repo diff --git a/.github/workflows/test-minimum-deps.yml b/.github/workflows/test-minimum-deps.yml index b5dfbcef5..1375064cb 100644 --- a/.github/workflows/test-minimum-deps.yml +++ b/.github/workflows/test-minimum-deps.yml @@ -20,7 +20,7 @@ jobs: defaults: run: - shell: bash -euo pipefail {0} + shell: bash -elo pipefail {0} steps: - name: Checkout repo