diff --git a/README_PYPI.md b/README_PYPI.md index af05bd5f..9b239926 100644 --- a/README_PYPI.md +++ b/README_PYPI.md @@ -95,6 +95,51 @@ levels deep — is one command: $ wppm -p ".[.]" -l9 ``` +## What did you actually ask for? + +`-p` and `-r` answer for one package. `-tl` answers for a whole list: it keeps only the +entries nothing else in that list already pulls in. Plainly, it prints that list and +nothing else -- so the output *is* the new file, and the counts go to stderr where a +redirection leaves them behind: + +```console +$ wppm requirements_slim.txt -tl -t D:\WPy64\python > requirements_slim_new.txt +# 160 entries -> 112 kept, 48 already pulled in +# 8 repeated, collapsed: brotli, openai, pympler, pytest, python-barcode, ... +``` + +Those two lines are commented although they go to stderr, so folding both streams into +one file (`> new.txt 2>&1`) still leaves a file pip can read. + +The notes the source file carried are kept, since they are its author's. `-v` adds the +reasoning: what the list is made from and when, the same counts, and every dropped entry +commented out with what pulls it in, so re-asking for one is uncommenting it. stderr then +keeps quiet, the counts being in the file already. + +```console +$ wppm requirements_slim.txt -tl -v -t D:\WPy64\python +# requirements_slim.txt, sorted, 2026-08-13 19:22:43 +# 160 entries -> 112 kept, 48 already pulled in +# 8 repeated, collapsed: brotli, openai, pympler, pytest, python-barcode, ... + +... +#numpy # <- baresql, clarabel, cvxpy, dask[array,dataframe,diagnostics], datashader, ... +#scikit-learn # <- imbalanced-learn, mlxtend, prince, skrub, umap-learn +#whatthepatch # <- spyder +``` + +With no file, the question becomes "of everything installed here, what did anything +actually ask for?": + +```console +$ wppm --top-level -t D:\WPy64\python +``` + +An optional dependency only counts where its extra is asked for, and a mutual pair keeps +both members -- dropping either would take the other with it. With `-ws` the facts come +from a wheelhouse instead of an installation, so a list can be pruned before anything is +built; where the wheelhouse holds several versions of a package, the newest one answers. + ## Everything is available as JSON Any of `-p`, `-r`, `-ls`, `-md` accepts `-j` / `--json`, so the same answers can gate a @@ -185,10 +230,10 @@ anything into it. ```text usage: wppm [-h] [-v] [--register] [--unregister] [--fix] [--movable] [-ws WHEELSOURCE] [-wd WHEELDRAIN] [-ls] [-lsa] [-md] [-p] [-r] - [-l LEVELS] [-j] [-t TARGET] [-i] [-u] + [-tl] [-l LEVELS] [-j] [-t TARGET] [-i] [-u] [package(s) or lockfile ...] -WinPython Package Manager: handle a Python distribution (WinPython or not) and its packages (17.10.20260808) +WinPython Package Manager: handle a Python distribution (WinPython or not) and its packages (17.11.20260813) positional arguments: package(s) or lockfile @@ -208,8 +253,9 @@ options: -md markdown summary of the installation -p show Package (!= missing) dependencies of the given package[option], [.]=all: wppm -p pandas[.] -r show Reverse (!= constraining) dependancies of the given package[option]: wppm -r pytest![test] + -tl, --top-level keep only the entries no other entry pulls in, sorted: wppm -tl, wppm requirements.txt -tl -v -l LEVELS show 'LEVELS' levels of dependencies (with -p, -r): wppm -p pandas -l1 - -j, --json machine-readable JSON output (with -p, -r, -ls, -md): wppm -p pandas[.] -j + -j, --json machine-readable JSON output (with -p, -r, -ls, -md, -tl): wppm -p pandas[.] -j -t TARGET path to target Python distribution (default: current environment) -i, --install install a given package wheel or pylock file (use pip for more features) -u, --uninstall uninstall package (use pip for more features) diff --git a/requirements64_slim.txt b/requirements64_slim.txt deleted file mode 100644 index 8282192e..00000000 --- a/requirements64_slim.txt +++ /dev/null @@ -1,524 +0,0 @@ -# 313 -# ortool 290MO UNCOMPRESSED -# re-add pandoc … 618Mo? -# 698 Mo after remoal of moviepy/imageio-ffmpeg -# 714 Mo after removal of maturin and WASM Klein -# 725Mo before removal of: maturin 7 Mo, WASM KLEIN example -#2024-09-22: swifter removed - -#2024-12-28 add pydantic_ai - -# the essential -wheel -pywin32 -build - - -# compilers -cython -pycparser -cffi - -# numeric stones -numpy - -scipy -sympy -Pillow -matplotlib - -pandas - -# sql - data - -pyodbc - - -SQLAlchemy -sqlparse -sqlite_bro -baresql -mysql_connector_python -pg8000 -ipython_sql - -XlsxWriter - -pymongo -redis - -# high numeric -#numexpr 2024-12-25: not worth it - -#h5py 2024-12-25: not worth it - -cytoolz -#netCDF4 2024-12-25: not worth it -xarray - -#Pulp 2024-12-25: not worth it - -scikit_learn -scikit_image - - - -# gui -jupyter -ipython - -spyder - -# seaborn wants patsy and statsmodels for linear modeling -seaborn -patsy -statsmodels - -holoviews -mpld3 - -# web -beautifulsoup4 - -lxml -html5lib -requests - - -simplejson - -flask - -# dev complements - -pytest -jedi -pep8 -pyflakes - -pylint - -numpydoc - -twine - -# other -networkx -nltk - -# PyAudio 2024-12-25: not worth it, no visibility of source -sounddevice - -pyserial - -#pdf - -reportlab - - -# Pierre Raybaut Stack (PyQt5 only for now) -plotpy -PythonQwt -guidata - -# for dask - -lmfit - -# Qt - -pyqtgraph - - -# yet other -julia - -# remember me why -certifi -click - - - -sphinx_rtd_theme -Sphinx -greenlet -rx - -# wheelhouse-uploader apache-libcloud (is heavy) - -Markdown - -prompt-toolkit -ptpython - -geopy - -wordcloud - - - -pycodestyle - -altair -nbconvert -pypandoc - - - - -fuzzywuzzy -#scikit_fuzzy -imageio - -#xlwings 2024-12-25 not worth it - -# parallelize (and replace celery) -joblib -#dask[complete] contains irrelevant pyarrow-hotfix -dask[array,dataframe,distributed,diagnostics] - -# dask 'bag' and 'delayed' -cloudpickle -toolz -partd - -brotli - - -pybind11 - -#SLIM_2024 pygame -plotnine - - - -#SLIM_2024 moviepy - - -#no download streamz - - -# Tensorflow_world -###edward -###Keras -###keras_vis -###Tensorflow_cpu -###tensorflow_probability - -##keras-tuner - -# pytorch eco-system -#Torch -#torchvision -#torchaudio -#botorch -#lightning -#kornia -#transformers[torch] -#accelerate -#fidle (tensorboard-2.15.1 wants still protobuf<4.24 , problem with ortools) -## waiting for torch: -##fastai -##spacy - -# if we drop torch: jaxlib ml-dtypes numpy scipy jax opt-einsum -#jax[cpu] - -terminado - -# pywinpty added due link removed from terminado for PyPy3 -pywinpty - -Send2Trash - -vega_datasets - -regex - -#loky - -hvplot - -#clrmagic 2024-12-25 not Worth it -#pythonnet 2024-12-25 not Worth it - -cvxopt - -numba -##pyarrow - - -cvxpy -mypy - -datashader - -mlxtend - -##jupyterlab_rise - -simpy - - -trio - -#dead2025-07-13: imbalanced-learn then rewaken ? -imbalanced-learn - -tzlocal -astropy - -panel - -hypothesis -geopandas - -mercantile - -#rasterio 2024-12-25 not jangmin and maxsecure - -quantecon - -kiwisolver - -# automate notebooks 2019-04-26 -papermill - -autopep8 -black - -## winrt not yet - - -# python_language_server no more -python_lsp_server -pexpect - - -#swagger flask (still no asyncio choice) -#flask_accepts -#flaskerize -# flask_RESTplus ... shall be now flask-restx -quart -datasette -hypercorn - -#complementing asgi stack -#2023-03-19 no more with sqlalchemy2.0 -##databases[sqlite] - -## ibis-framework no big life - -folium -plotly - - -umap-learn -#SLIM_2024 virtualenv -pympler -## pipdeptree replaced par wppm - -##nlopt - -#2025-02-15 flask-sqlalchemy -#2025-02-15 flask-session -#2025-02-15 flask-Mail -python-dotenv -httpie -asgiref - -#SLIM_2024 importlib_metadata - - -##jupyterlab things -jupyterlab -widgetsnbextension - -##jupyterlab_launcher -jupyter_bokeh -#2023-05-21 too fragile: dask_labextension -#SLIM_2024 pydeck - -## jupyterlab3 only -jupyterlab-widgets - -## 2020-09-27 jupyterlab2 only -ipympl -ipyleaflet - -ipycanvas - -#SLIM_2024 wasmer -#SLIM_2024 wasmer_compiler_cranelift -#SLIM_2024 wasmer_compiler_singlepass - -fastapi - - -datasette_graphql -sqlite_utils -aiosqlite - - -## onnxruntime - -##sklearn-contrib-lightning -openpyxl -zstandard - - -pynndescent - - -flit -## 2023-10-15 (too constraining) poetry -## so moves to hatch -## buth hatch wants now uv.. a bit too much -hatchling - -#ecos 2024-12-25 - -##csvs_to_sqlite -datasette_graphql -sqlite_utils - -maturin - -orjson - - -#2023-08-22 fuzz replacements -rapidfuzz - -streamlit -streamlit-bokeh - -## 2024-09-08 until plotly at least updates its lumnio thing https://github.com/plotly/plotly.py/pull/4685 -# dash -alembic - - -#webapps example needs -Django -#channels - - -## badly formed uvicorn[standard] -uvicorn -python-multipart -deap - -polars - -##timseries bis - - -# for SSRS -requests_ntlm -missingno - -##xgboost -# lightgbm ? -duckdb - - -# for flask -waitress - -#2023-10-08 soon:jupyterlab-lsp - -array-api-compat - - -mpmath - -openai - - -#azure -azure-identity -azure-cosmos -azure-core - -## build - -# write & read QRcode -python-barcode -qrcode -#not compatible with numpy-2.3 2025-09-14: opencv-python - -#llm follow-up -# llm -#risk llm_gpt4all -# llm_llama_cpp -# llm_markov -# llm-python -huggingface_hub - - -#course of langchain https://learn.deeplearning.ai/langchain-chat-with-your-data -pypdf -yt_dlp -pydub - - -# new friends of panel (load psygnal, a qt-like signaling) -anywidget - -keras - - - -# pyomo becomes a frequent wrapper nowodays - -langchain -pyomo -#NUMPY2_WAIT highspy -#pymoo 2024-12-25 not well maintained - -clarabel -#SLIM_2024 ortools - -#waiting cvxpy -scs -optuna - - -#pyarrow complement -adbc_driver_manager -#adbc_driver_sqlite -#backport optional importlib_resources - -#cartopy 2024-12-26 use geopandas more popular (that include gdal via pyogrio) - -#2024-12-26 agent ai things -pydantic-ai-slim[a2a,anthropic,cli,cohere,evals,google,groq,mcp,mistral,openai,vertexai] -skrub -termcolor -tiktoken - -# from Microsoft own distro -thefuzz -tabulate -squarify -PyWavelets -prince -faker - -pyusb - -sv-ttk -typer - -onnxruntime -markitdown -pyvisa - -soundfile -psycopg2 -pipdeptree - -ipykernel!=7.0.0,!=7.0.1 - -foundry-local-sdk - diff --git a/requirements_slim.txt b/requirements_slim.txt new file mode 100644 index 00000000..24e9b326 --- /dev/null +++ b/requirements_slim.txt @@ -0,0 +1,179 @@ +# requirements_slim.txt, sorted, 2026-08-13 19:15:35 +# 160 entries -> 112 kept, 48 already pulled in +# 8 repeated, collapsed: brotli, openai, pympler, pytest, python-barcode, pywavelets, sympy, xlsxwriter + +adbc_driver_manager +aiosqlite +anywidget +array-api-compat +azure-cosmos +baresql +brotli +build +cvxopt +cvxpy +cython +cytoolz +dask[array,dataframe,diagnostics] +datasette_graphql +datashader +deap +duckdb +faker +fastapi +flit +folium +foundry-local-sdk +fuzzywuzzy +geopy +hatchling +html5lib +httpie +hvplot +hypothesis +imbalanced-learn +ipycanvas +ipyleaflet +ipympl +ipython_sql +julia +jupyter +jupyter_bokeh +keras +langchain +lmfit +lxml +markitdown +maturin +mercantile +missingno +mlxtend +mpld3 +mssql-python +mypy[mypyc] +mysql_connector_python +nltk +onnxruntime_genai +opencv-python +openpyxl +optuna +papermill +pep8 +pg8000 +pipdeptree +plotly +plotnine +plotpy +polars +prince +psycopg2 +ptpython +pybind11 +pydantic-ai-slim[a2a,anthropic,cli,cohere,evals,google,groq,mcp,mistral,openai,vertexai]==1.12.0 +pympler +pyodbc +pyomo +pypandoc +pypdf +pyqtgraph +pyserial +pytest +python-barcode +pyusb +pyvisa +pywavelets +qrcode +quantecon +quart +redis +reportlab +requests_ntlm +rx +shapely +simplejson +simpy +skrub +sounddevice +soundfile +sphinx_rtd_theme +spyder +sqlite_bro +squarify +streamlit-bokeh +termcolor +thefuzz +tiktoken +trio +twine +typer +tzlocal +umap-learn +vega_datasets +waitress +wheel +wordcloud +xlsxwriter +yt_dlp + +# ---- already pulled in by an entry above ---- +#alembic # <- optuna +#Altair # <- prince, streamlit, streamlit-bokeh +#appdirs # <- ptpython +#azure-core # <- azure-cosmos, azure-identity, mssql-python +#azure-identity # <- mssql-python +#black # <- spyder +#clarabel # <- cvxpy +#datasette # <- datasette-graphql +#fasta2a # <- pydantic-ai-slim +#flatbuffers # <- markitdown, onnxruntime, onnxruntime-genai +#groq # <- pydantic-ai-slim +#guidata # <- PlotPy +#holoviews # <- hvplot +#huggingface_hub # <- pydantic-ai-slim +#imageio # <- PlotPy, scikit-image +#jupyterlab # <- jupyter +#markdownify # <- markitdown +#matplotlib # <- ipympl, missingno, mlxtend, mpld3, plotnine, ... +#mistralai # <- pydantic-ai-slim +#networkx # <- PlotPy, scikit-image +#numba # <- datashader, quantecon, umap-learn +#numpy # <- baresql, clarabel, cvxpy, dask, datashader, ... +#onnxruntime # <- markitdown, onnxruntime-genai +#openai # <- pydantic-ai-slim +#pandas # <- baresql, dask, datashader, holoviews, hvplot, ... +#protobuf # <- markitdown, onnxruntime, onnxruntime-genai, streamlit, streamlit-bokeh +#pyct # <- datashader +#pydeck # <- streamlit, streamlit-bokeh +#pysocks # <- httpie +#PythonQwt # <- PlotPy +#pywin32 # <- pydantic-ai-slim, Pympler +#pyzmq # <- jupyter, jupyterlab, papermill, spyder +#requests # <- azure-core, azure-cosmos, azure-identity, datashader, flit, ... +#scikit-image # <- PlotPy +#scikit-learn # <- imbalanced-learn, mlxtend, prince, skrub, umap-learn +#scipy # <- clarabel, cvxpy, datashader, imbalanced-learn, lmfit, ... +#scs # <- cvxpy +#seaborn # <- missingno +#setuptools # <- cvxpy, datasette, datasette-graphql, httpie, mypy +#SQLalchemy # <- alembic, ipython-sql, optuna +#sqlite-utils # <- datasette-graphql +#statsmodels # <- plotnine +#streamlit # <- streamlit-bokeh +#sympy # <- quantecon +#tifffile # <- PlotPy, scikit-image +#websockets # <- langchain, pydantic-ai-slim, streamlit, streamlit-bokeh +#whatthepatch # <- spyder +#zstandard # <- langchain + +# ---- notes kept from the source ---- +# for WinPython-3.14 slim +# 2026-07-12 +# switch geopandas for opencv +# 2026-08-03: +# astropy, django, distributed removed (place) +# pymongo removed +# 2027-08-07 +# remove geopandas (and so pyogrio, pyproj) +# 2026-08-08 +# pydub unmaintained + diff --git a/tests/test_piptree.py b/tests/test_piptree.py index cd5afd71..2ac48202 100644 --- a/tests/test_piptree.py +++ b/tests/test_piptree.py @@ -118,6 +118,41 @@ def test_environment_exposes_every_marker_field_packaging_needs(self, pip): } +class TestWheelhouse: + """Reading a directory of wheels instead of an installation.""" + + @pytest.fixture + def wheelhouse(self, tmp_path): + import zipfile + house = tmp_path / "wheels" + house.mkdir() + + def wheel(name, version, requires=()): + with zipfile.ZipFile(house / f"{name}-{version}-py3-none-any.whl", "w") as zf: + lines = ["Metadata-Version: 2.1", f"Name: {name}", f"Version: {version}"] + lines += [f"Requires-Dist: {r}" for r in requires] + zf.writestr(f"{name}-{version}.dist-info/METADATA", "\n".join(lines) + "\n") + + wheel("solo", "1.0") + wheel("twice", "1.0", requires=["solo"]) + wheel("twice", "2.0") + (house / "not-a-package.tar.gz").write_bytes(b"not a tarball at all") + return house + + def test_an_unreadable_archive_does_not_lose_the_others(self, wheelhouse): + pip = piptree.PipData(None, str(wheelhouse)) + assert {"solo", "twice"} <= set(pip.distro) + + def test_only_the_newest_version_of_a_package_is_kept(self, wheelhouse): + pip = piptree.PipData(None, str(wheelhouse)) + assert pip.distro["twice"]["version"] == "2.0" + + def test_the_newest_version_brings_its_own_dependencies(self, wheelhouse): + """twice 1.0 needs solo, twice 2.0 does not: the answer must be 2.0's.""" + pip = piptree.PipData(None, str(wheelhouse)) + assert pip.dependency_closure("twice") == set() + + class TestCycles: def test_mutual_dependency_terminates(self, tmp_path): """A <-> B must not recurse forever.""" diff --git a/tests/test_top_level.py b/tests/test_top_level.py new file mode 100644 index 00000000..6a2a2bb5 --- /dev/null +++ b/tests/test_top_level.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +"""--top-level: keep only the entries no other entry already pulls in. + +Same synthetic site-packages trick as test_piptree.py -- the point is the +graph, not the packages. +""" +import json +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +from wppm import piptree, utils, wppm as wppm_module + +from conftest import write_dist, windows_only + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +@pytest.fixture +def graph(tmp_path): + """app -> lib -> helper, app[fancy] -> fancylib, and a standalone orphan.""" + root = tmp_path / "dist" + site = root / "Lib" / "site-packages" + site.mkdir(parents=True) + (root / "python.exe").write_bytes(b"MZ") + write_dist(site, "app", "1.0", requires=["lib>=1.0", 'fancylib; extra == "fancy"'], + extras=["fancy"]) + write_dist(site, "lib", "1.5", requires=["helper"]) + write_dist(site, "helper", "0.3") + write_dist(site, "fancylib", "2.0") + write_dist(site, "orphan", "9.9") + return root + + +@pytest.fixture +def pip(graph): + return piptree.PipData(str(graph)) + + +class TestSplitRequirement: + @pytest.mark.parametrize("text, expected", [ + ("numpy", ("numpy", [])), + ("numpy==2.0", ("numpy", [])), + ("numpy >= 2.0", ("numpy", [])), + ("Pillow", ("pillow", [])), + ("mypy[mypyc]", ("mypy", ["mypyc"])), + ("dask[array,dataframe]>=2.0", ("dask", ["array", "dataframe"])), + ("scipy; python_version > '3.10'", ("scipy", [])), + ]) + def test_parses(self, text, expected): + assert piptree.PipData.split_requirement(text) == expected + + +class TestClosure: + def test_follows_the_chain(self, pip): + assert pip.dependency_closure("app") == {"lib", "helper"} + + def test_ignores_an_extra_nobody_asked_for(self, pip): + assert "fancylib" not in pip.dependency_closure("app") + + def test_follows_an_extra_that_is_asked_for(self, pip): + assert "fancylib" in pip.dependency_closure("app", "fancy") + + def test_leaf_reaches_nothing(self, pip): + assert pip.dependency_closure("helper") == set() + + +class TestTopLevel: + def test_installed_set_keeps_only_what_nothing_requires(self, pip): + assert pip.top_level()["kept"] == ["app", "fancylib", "orphan"] + + def test_drops_an_entry_another_entry_pulls_in(self, pip): + result = pip.top_level(["app", "lib", "orphan"]) + assert result["kept"] == ["app", "orphan"] + assert result["dropped"] == {"lib": ["app"]} + + def test_names_every_puller_of_a_dropped_entry(self, pip): + assert pip.top_level(["app", "lib", "helper"])["dropped"]["helper"] == ["app", "lib"] + + def test_keeps_an_entry_whose_puller_is_not_listed(self, pip): + """Nothing listed pulls helper in, so it stays.""" + assert pip.top_level(["helper", "orphan"])["kept"] == ["helper", "orphan"] + + def test_an_extra_only_dependency_stays_unless_the_extra_is_asked_for(self, pip): + assert "fancylib" in pip.top_level(["app", "fancylib"])["kept"] + assert "fancylib" in pip.top_level(["app[fancy]", "fancylib"])["dropped"] + + def test_keeps_the_entry_as_written(self, pip): + assert pip.top_level(["app[fancy]", "lib"])["kept"] == ["app[fancy]"] + + def test_reports_a_repeated_entry_once(self, pip): + result = pip.top_level(["orphan", "orphan"]) + assert result["kept"] == ["orphan"] + assert result["duplicates"] == ["orphan"] + + def test_a_repeat_keeps_the_fuller_spelling(self, pip): + assert pip.top_level(["app", "app[fancy]"])["kept"] == ["app[fancy]"] + + def test_an_entry_the_target_lacks_is_kept_and_reported(self, pip): + result = pip.top_level(["orphan", "nosuchpackage"]) + assert result["unknown"] == ["nosuchpackage"] + assert "nosuchpackage" in result["kept"] + + def test_sorting_is_case_insensitive(self, pip): + assert pip.top_level(["orphan", "App", "fancylib"])["kept"] == ["App", "fancylib", "orphan"] + + def test_empty_input_gives_empty_output(self, pip): + assert pip.top_level([]) == {"kept": [], "dropped": {}, "duplicates": [], "unknown": []} + + +class TestMutualDependency: + @pytest.fixture + def cycle(self, tmp_path): + root = tmp_path / "cyc" + site = root / "Lib" / "site-packages" + site.mkdir(parents=True) + (root / "python.exe").write_bytes(b"MZ") + write_dist(site, "aaa", "1.0", requires=["bbb"]) + write_dist(site, "bbb", "1.0", requires=["aaa"]) + write_dist(site, "ccc", "1.0", requires=["aaa"]) + return piptree.PipData(str(root)) + + def test_a_mutual_pair_keeps_both(self, cycle): + """Dropping either would take the other with it.""" + assert cycle.top_level(["aaa", "bbb"])["kept"] == ["aaa", "bbb"] + + def test_something_outside_the_cycle_still_drops_it(self, cycle): + result = cycle.top_level(["aaa", "bbb", "ccc"]) + assert result["kept"] == ["ccc"] + assert set(result["dropped"]) == {"aaa", "bbb"} + + +class TestRendering: + def test_plain_output_is_the_list_and_nothing_else(self, pip): + """Redirect it and what lands in the file is the file.""" + assert wppm_module.top_level_as_requirements(pip.top_level(["app", "lib"])) == ["app"] + + def test_verbose_comments_out_what_went_and_why(self, pip): + lines = wppm_module.top_level_as_requirements(pip.top_level(["app", "lib"]), verbose=True) + assert "app" in lines + assert "#lib # <- app" in lines + + def test_verbose_heads_the_list_with_source_and_time(self, pip): + lines = wppm_module.top_level_as_requirements( + pip.top_level(["app", "lib"]), source="req.txt", verbose=True) + assert re.fullmatch(r"# req\.txt, sorted, \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}", lines[0]) + + def test_verbose_heads_the_list_with_the_same_counts_stderr_gives(self, pip): + result = pip.top_level(["app", "lib", "orphan"]) + lines = wppm_module.top_level_as_requirements(result, verbose=True) + assert lines[1:2] == wppm_module.top_level_summary(result)[:1] + assert lines[1] == "# 3 entries -> 2 kept, 1 already pulled in" + + def test_source_notes_are_kept_even_plainly(self, pip): + """They are the author's own lines, not our commentary.""" + lines = wppm_module.top_level_as_requirements(pip.top_level(["app"]), comments=["# a note"]) + assert "# a note" in lines + + +class TestSummary: + def test_counts_what_happened(self, pip): + notes = wppm_module.top_level_summary(pip.top_level(["app", "lib", "orphan"])) + assert notes[0] == "# 3 entries -> 2 kept, 1 already pulled in" + + def test_every_note_is_a_comment(self, pip): + """stdout and stderr may well end up in the same file.""" + notes = wppm_module.top_level_summary(pip.top_level(["orphan", "orphan", "nosuchpackage"])) + assert len(notes) == 3 + assert all(note.startswith("# ") for note in notes) + + def test_reports_repeats(self, pip): + notes = wppm_module.top_level_summary(pip.top_level(["orphan", "orphan"])) + assert any("repeated" in note for note in notes) + + def test_reports_what_the_target_lacks(self, pip): + notes = wppm_module.top_level_summary(pip.top_level(["orphan", "nosuchpackage"])) + assert any("nosuchpackage" in note for note in notes) + + +class TestReadRequirements: + def test_splits_entries_from_comments(self, tmp_path): + path = tmp_path / "r.txt" + path.write_text("# a note\n\nnumpy\n pandas \n#disabled\n", encoding="utf-8") + assert utils.read_requirements(path) == (["numpy", "pandas"], ["# a note", "#disabled"]) + + +@windows_only +class TestCli: + def run(self, *args): + proc = subprocess.run( + [sys.executable, "-X", "utf8", "-m", "wppm", *args], + capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=300, + encoding="utf-8", errors="replace", + ) + assert proc.returncode == 0, f"exit {proc.returncode}\n{proc.stdout}\n{proc.stderr}" + return proc + + def wppm(self, *args): + return self.run(*args).stdout + + def test_top_level_of_a_target(self, graph): + assert self.wppm("-t", str(graph), "--top-level").splitlines() == ["app", "fancylib", "orphan"] + + def test_the_short_flag_does_the_same(self, graph): + assert self.wppm("-t", str(graph), "-tl") == self.wppm("-t", str(graph), "--top-level") + + def test_verbose_adds_the_reasoning(self, graph): + assert "#lib # <- app" in self.wppm("-t", str(graph), "--top-level", "-v").splitlines() + + def test_verbose_does_not_say_the_counts_twice(self, graph): + """Under -v they head the list, so stderr keeps quiet.""" + proc = self.run("-t", str(graph), "--top-level", "-v") + assert "entries ->" in proc.stdout + assert "entries ->" not in proc.stderr + + def test_top_level_of_a_requirements_file(self, graph, tmp_path): + req = tmp_path / "req.txt" + req.write_text("# keep me\nlib\napp\n", encoding="utf-8") + out = self.wppm("-t", str(graph), str(req), "--top-level").splitlines() + assert "app" in out + assert "#lib" not in out + assert "# keep me" in out + + def test_the_counts_go_to_stderr_not_into_the_list(self, graph, tmp_path): + req = tmp_path / "req.txt" + req.write_text("lib\napp\n", encoding="utf-8") + proc = self.run("-t", str(graph), str(req), "--top-level") + assert proc.stdout.splitlines() == ["app"] + assert "2 entries -> 1 kept" in proc.stderr + + def test_json_output_parses(self, graph): + data = json.loads(self.wppm("-t", str(graph), "--top-level", "-j")) + assert set(data) == {"kept", "dropped", "duplicates", "unknown"} + assert data["kept"] == ["app", "fancylib", "orphan"] diff --git a/wppm/__init__.py b/wppm/__init__.py index 9d328227..ef24d29d 100644 --- a/wppm/__init__.py +++ b/wppm/__init__.py @@ -28,6 +28,6 @@ OTHER DEALINGS IN THE SOFTWARE. """ -__version__ = '17.10.20260808' +__version__ = '17.11.20260813' __license__ = __doc__ __project_url__ = 'http://winpython.github.io/' diff --git a/wppm/packagemetadata.py b/wppm/packagemetadata.py index c82efd91..80257330 100644 --- a/wppm/packagemetadata.py +++ b/wppm/packagemetadata.py @@ -40,17 +40,20 @@ def get_installed_metadata(path = None) -> List[PackageMetadata]: return pkgs def get_directory_metadata(directory: str) -> List[PackageMetadata]: - # For each .whl/.tar.gz file in directory, extract metadata + """Metadata of every wheel and sdist in *directory*. + + An archive that carries no metadata (a plain source tarball, say) is + skipped: a wheelhouse holds what it holds, and one odd file in it must not + cost the caller the other three thousand. + """ pkgs = [] for fname in os.listdir(directory): - if fname.endswith('.whl'): - # Extract METADATA from wheel - meta = extract_metadata_from_wheel(os.path.join(directory, fname)) - pkgs.append(meta) - elif fname.endswith('.tar.gz'): - # Extract PKG-INFO from sdist - meta = extract_metadata_from_sdist(os.path.join(directory, fname)) - pkgs.append(meta) + extract = extract_metadata_from_wheel if fname.endswith('.whl') else extract_metadata_from_sdist + if fname.endswith(('.whl', '.tar.gz')): + try: + pkgs.append(extract(os.path.join(directory, fname))) + except (ValueError, KeyError, OSError, tarfile.TarError, zipfile.BadZipFile) as e: + print(f"skipped {fname}: {e}", file=sys.stderr) return pkgs def extract_metadata_from_wheel(path: str) -> PackageMetadata: diff --git a/wppm/piptree.py b/wppm/piptree.py index e6a51e35..9ee1fbab 100644 --- a/wppm/piptree.py +++ b/wppm/piptree.py @@ -22,8 +22,10 @@ from typing import Dict, List, Optional, Tuple, Union try: from packaging.markers import Marker + from packaging.version import Version except ModuleNotFoundError: from pip._vendor.packaging.markers import Marker + from pip._vendor.packaging.version import Version from importlib.metadata import Distribution, distributions from pathlib import Path from . import utils @@ -96,7 +98,7 @@ def _get_environment(self) -> Dict[str, str]: def _get_packages(self, search_path: str, wheelhouse) -> List[Distribution]: """Retrieve installed packages from the specified path.""" if wheelhouse: - return pm.get_directory_metadata(wheelhouse) + return self._newest_of_each(pm.get_directory_metadata(wheelhouse)) if sys.executable == search_path: return pm.get_installed_metadata() #Distribution.discover() else: @@ -104,6 +106,27 @@ def _get_packages(self, search_path: str, wheelhouse) -> List[Distribution]: # with the venv layout (python.exe in Scripts, site-packages at the root) return pm.get_installed_metadata(path=[utils.get_site_packages_path(search_path)]) + @staticmethod + def _newest_of_each(packages: List) -> List: + """One distribution per name, the highest version. + + A wheelhouse commonly holds several versions of a package; the + environment PipData models holds one, and reading whichever the + directory listing returned last would make the answer arbitrary. + """ + def version_of(package): + try: + return Version(package.version) + except Exception: # a local or malformed version still sorts, just last + return Version("0") + + newest = {} + for package in packages: + key = PipData.normalize(package.name) + if key not in newest or version_of(package) > version_of(newest[key]): + newest[key] = package + return list(newest.values()) + def _process_packages(self, packages: List[Distribution]) -> None: """Process packages metadata and store them in the distro dictionary.""" for package in packages: @@ -298,6 +321,81 @@ def _get_dependency_tree(self, package_name: str, extra: str = "", version_req: else: return [] + @staticmethod + def split_requirement(text: str) -> Tuple[str, List[str]]: + """'dask[array,dataframe]>=2.0' -> ('dask', ['array', 'dataframe']).""" + name_extras = re.split(r"[=<>~!;@ ]", text.strip(), maxsplit=1)[0] + name, _, extras = name_extras.partition("[") + return PipData.normalize(name), [e.strip() for e in extras.rstrip("]").split(",") if e.strip()] + + def dependency_closure(self, package: str, extra: str = "") -> set: + """Every installed package reachable from `package[extra]`, itself excluded. + + An optional dependency counts only where its extra is asked for: a + requirement gated on `extra == "test"` is not followed for a bare + package, since nothing installed it on that account. + """ + key = self.normalize(package) + reached, seen = set(), set() + stack = [(key, e) for e in extra.split(",") if e] + [(key, "")] + while stack: + pkg, pkg_extra = stack.pop() + if (pkg, pkg_extra) in seen or pkg not in self.distro: + continue + seen.add((pkg, pkg_extra)) + for req in self.distro[pkg]["requires_dist"]: + marker = req.get("req_marker") + if marker and not self._marker_true(marker, pkg_extra): + continue + if req["req_key"] in self.distro: + reached.add(req["req_key"]) + stack.append((req["req_key"], req["req_extra"])) + reached.discard(key) + return reached + + def top_level(self, entries: Optional[List[str]] = None) -> Dict: + """Which entries no other entry already pulls in. + + `entries` are requirement strings ("dask[array]", "numpy==2.0"); given + none, every installed package is an entry. Returns the entries to keep + in alphabetical order, and for each dropped one the entries that pull + it in -- a requirements file saying what it means, rather than what a + dependency would have installed anyway. + + A mutual pair (a needs b, b needs a) keeps both: dropping either would + take the other with it. An entry the target has never installed cannot + be resolved, so it is kept and reported apart. + """ + if entries is None: + entries = [pkg["name"] for pkg in self.distro.values()] + asked: Dict[str, Tuple[str, List[str]]] = {} # key -> (as written, extras) + duplicates = [] + for text in entries: + key, extras = self.split_requirement(text) + if key in asked: + duplicates.append(text) + if len(text) <= len(asked[key][0]): + continue # keep the fuller spelling: extras and pins matter + asked[key] = (text, extras) + reach = {key: self.dependency_closure(key, ",".join(extras)) if key in self.distro else set() + for key, (_, extras) in asked.items()} + dropped = {} + for key, (text, _) in asked.items(): + # named plainly: what pulls a package in is the package, not the + # extras and pin the entry happened to be written with + pullers = sorted((self.distro[other]["name"] if other in self.distro else other + for other in asked + if other != key and key in reach[other] and other not in reach[key]), + key=str.lower) + if pullers: + dropped[text] = pullers + return { + "kept": sorted((text for text, _ in asked.values() if text not in dropped), key=str.lower), + "dropped": dict(sorted(dropped.items(), key=lambda item: item[0].lower())), + "duplicates": sorted(duplicates, key=str.lower), + "unknown": sorted((text for key, (text, _) in asked.items() if key not in self.distro), key=str.lower), + } + def _node_text(self, node: Dict) -> str: """Render a tree node dict as its one-line text form.""" if not node["installed"]: diff --git a/wppm/utils.py b/wppm/utils.py index f5c9d9a0..96a8a4c4 100644 --- a/wppm/utils.py +++ b/wppm/utils.py @@ -53,6 +53,15 @@ def get_site_packages_path(path=None): """Return the path to the Python site-packages directory.""" return str(get_install_root(path) / 'Lib' / 'site-packages') +def read_requirements(path): + """Split a pip requirements file into its entries and its comment lines.""" + entries, comments = [], [] + for line in Path(path).read_text(encoding='utf-8').splitlines(): + text = line.strip() + if text: + (comments if text.startswith('#') else entries).append(text) + return entries, comments + def first_line(text, default="?"): """Return the first non-empty line of *text*, or *default* if there is none.""" for line in text.splitlines(): diff --git a/wppm/wppm.py b/wppm/wppm.py index deae0e3b..ac907db8 100644 --- a/wppm/wppm.py +++ b/wppm/wppm.py @@ -12,6 +12,7 @@ import shutil import subprocess import json +from datetime import datetime from pathlib import Path from argparse import ArgumentParser, RawTextHelpFormatter from . import utils, piptree, diff, __version__ @@ -277,6 +278,51 @@ def install_bdist_direct(self, package, install_options=None): package = Package(fname) self._print_done() +def few(names, limit=6): + """First few names of a list, and how many more there were.""" + return ", ".join(names[:limit]) + (f", and {len(names) - limit} more" if len(names) > limit else "") + +def top_level_as_requirements(result, comments=(), source=None, verbose=False): + """Render piptree.top_level() as the requirements file it proposes. + + Plainly, it is that file and nothing else: the entries, plus the notes the + source itself carried, so redirecting the output replaces the source + without losing what its author wrote in it. -v adds the reasoning -- what + the list is made from and when, the same counts stderr gives, and every + dropped entry commented out with what pulls it in, so re-asking for one is + uncommenting it. + """ + lines = [] + if verbose: + lines += [f"# {Path(source).name if source else 'installed packages'}, sorted," + f" {datetime.now():%Y-%m-%d %H:%M:%S}"] + lines += top_level_summary(result) + [""] + lines += result["kept"] + dropped = result["dropped"] + if verbose and dropped: + lines += ["", "# ---- already pulled in by an entry above ----"] + for text, pullers in dropped.items(): + lines.append(f"#{text} # <- {', '.join(pullers[:5])}{', ...' if len(pullers) > 5 else ''}") + if comments: + lines += ["", "# ---- notes kept from the source ----"] + list(comments) + return lines + +def top_level_summary(result): + """What the caller should know about the answer, rather than of it. + + Goes to stderr, so a redirected list stays a list -- and heads the list + itself under -v, where the reasoning belongs in the file. Commented either + way: someone will fold the two streams into one file sooner or later, and a + comment costs nothing. + """ + kept, dropped = result["kept"], result["dropped"] + notes = [f"{len(kept) + len(dropped)} entries -> {len(kept)} kept, {len(dropped)} already pulled in"] + if result["duplicates"]: + notes.append(f"{len(result['duplicates'])} repeated, collapsed: {few(result['duplicates'], 12)}") + if result["unknown"]: + notes.append(f"{len(result['unknown'])} not installed in the target, so left unresolved: {few(result['unknown'])}") + return [f"# {note}" for note in notes] + def main(test=False): # package summaries may contain characters the console codepage can't encode (emoji): don't crash if sys.stdout and hasattr(sys.stdout, "reconfigure"): @@ -301,8 +347,9 @@ def main(test=False): parser.add_argument("-md", dest="markdown", action="store_true",help=f"markdown summary of the installation") parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package (!= missing) dependencies of the given package[option], [.]=all: wppm -p pandas[.]") parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse (!= constraining) dependancies of the given package[option]: wppm -r pytest![test]") + parser.add_argument("-tl", "--top-level", action="store_true", help="keep only the entries no other entry pulls in, sorted: wppm -tl, wppm requirements.txt -tl -v") parser.add_argument("-l", dest="levels", type=int, default=-1, help="show 'LEVELS' levels of dependencies (with -p, -r): wppm -p pandas -l1") - parser.add_argument("-j", "--json", dest="json", action="store_true", help="machine-readable JSON output (with -p, -r, -ls, -md): wppm -p pandas[.] -j") + parser.add_argument("-j", "--json", dest="json", action="store_true", help="machine-readable JSON output (with -p, -r, -ls, -md, -tl): wppm -p pandas[.] -j") parser.add_argument("-t", dest="target", default=sys.prefix, help=f'path to target Python distribution (default: "{sys.prefix}")') parser.add_argument("-i", "--install", action="store_true", help="install a given package wheel or pylock file (use pip for more features)") parser.add_argument("-u", "--uninstall", action="store_true", help="uninstall package (use pip for more features)") @@ -331,6 +378,20 @@ def main(test=False): pack, extra, *other = (args_fname + "[").replace("]", "[").split("[") print(pip.up(pack, extra, args.levels if args.levels>=0 else 1, verbose=args.verbose, format="json" if args.json else "text")) sys.exit() + elif args.top_level: + pip = piptree.PipData(targetpython, args.wheelsource) + source = next((Path(f) for f in args.fname if f and Path(f).is_file()), None) + entries, comments = utils.read_requirements(source) if source else (None, []) + result = pip.top_level(entries) + if args.json: + print(json.dumps(result, indent=4)) + sys.exit() + for line in top_level_as_requirements(result, comments, source, args.verbose): + print(line) + if not args.verbose: # -v already heads the list with them; don't say it twice + for note in top_level_summary(result): + print(note, file=sys.stderr) + sys.exit() elif args.list: pip = piptree.PipData(targetpython, args.wheelsource) todo= []