diff --git a/build_winpython_meta.py b/build_winpython_meta.py index 611dbf3b..e90bfb74 100644 --- a/build_winpython_meta.py +++ b/build_winpython_meta.py @@ -6,28 +6,76 @@ import subprocess from pathlib import Path +# Flavor paths are names, not full paths: these say what they hang from. +UNDER_BASEDIR = ("requirements", "source_dirs", "wheelhousereq") +UNDER_ROOT = ("toolsdirs",) + def load_builds(config_file): with open(config_file, "rb") as f: config = tomllib.load(f) - builds = config["builds"] python_versions = config.get("pythons", {}) + # A file may still spell out one [[builds]] block per build; otherwise the + # builds are the (python, flavor) pairs [pythons] asks for. + builds = config["builds"] if "builds" in config else expand_builds(config) return builds, python_versions -def declared_file(build, key, default): - """Path named by the build for `key`, else `default`. +def expand_builds(config): + """One build per flavor listed by each [pythons."3XX"], paths derived. - A path a build spells out must exist: pip_install() skips a missing - requirements file without failing, which would drop the packages silently. + A flavor names its files relative to the build directory, so a new Python + minor is one [pythons] block. Where a Python needs its own file - a + requirements list under another name, say - [pythons."3XX".overrides.flavor] + replaces that flavor's entries for that Python alone. """ + defaults = config.get("defaults", {}) + flavors = config.get("flavors", {}) + builds = [] + for target, vinfo in config.get("pythons", {}).items(): + root = Path(vinfo.get("root_dir_for_builds", defaults.get("root_dir_for_builds", ""))) + basedir = root / f"bd{target}" + overrides = vinfo.get("overrides", {}) + for flavor in vinfo.get("builds", []): + if flavor not in flavors: + raise KeyError(f"python {target} builds {flavor!r}, which has no [flavors.{flavor}]") + build = {**defaults, "name": flavor, "python_target": target, "flavor": flavor} + for key, value in {**flavors[flavor], **overrides.get(flavor, {})}.items(): + if key in UNDER_BASEDIR: + value = str(basedir / value) + elif key in UNDER_ROOT: + value = str(root / value) + build[key] = value + builds.append(build) + return builds + +def select_builds(builds, wanted): + """Builds asked for on the command line: "315", "315:slim", ":slim".""" + if not wanted: + return builds + kept = [] + for spec in wanted: + target, _, flavor = spec.partition(":") + matching = [b for b in builds + if (not target or b["python_target"] == target) + and (not flavor or b["flavor"] == flavor)] + if not matching: + raise SystemExit(f"no build matches {spec!r}") + kept += [b for b in matching if b not in kept] + return kept + +def must_exist(path, build, key): + """pip_install() skips a missing requirements file without failing, which + would drop those packages silently. Say so instead.""" + if path and not Path(path).exists(): + raise FileNotFoundError(f"build {build['name']!r} needs {key} = {path!r}, which does not exist") + return path + +def declared_file(build, key, default): + """Path named by the build for `key`, else `default`.""" declared = build.get(key) - if declared is None: - return str(default) - if not Path(declared).exists(): - raise FileNotFoundError(f"build {build['name']!r} sets {key} = {declared!r}, which does not exist") - return str(declared) - -def run_build(build, python_versions): - print(f"\n=== Building WinPython: {build['name']} ===") + return str(default) if declared is None else must_exist(str(declared), build, key) + +def run_build(build, python_versions, dry_run=False): + print(f"\n=== Building WinPython: {build['python_target']} {build['name']} ===") print(build) root_dir_for_builds = build["root_dir_for_builds"] @@ -35,11 +83,11 @@ def run_build(build, python_versions): my_flavor = build["flavor"] my_arch = str(build["arch"]) my_create_installer = build.get("create_installer", "True") - my_requirements = build.get("requirements", "") + my_requirements = must_exist(build.get("requirements", ""), build, "requirements") my_source_dirs = build.get("source_dirs", "") my_find_links = build.get("find_links", "") my_toolsdirs = build.get("toolsdirs", "") - wheelhousereq = build.get("wheelhousereq", "") + wheelhousereq = must_exist(build.get("wheelhousereq", ""), build, "wheelhousereq") # "pip" (default, what ships) | "none" | "parallel" | "parallel-N" my_bytecode = build.get("bytecode", "pip") @@ -89,15 +137,18 @@ def run_build(build, python_versions): "--create-installer", my_create_installer, ] - print("Running build command:") + print("Dry run, build command:" if dry_run else "Running build command:") print(" ".join(build_cmd)) - subprocess.run(build_cmd, cwd=os.getcwd(), check=False) + if not dry_run: + subprocess.run(build_cmd, check=False) def main(): - config_file = sys.argv[1] if len(sys.argv) > 1 else "winpython_buildsNOT.toml" + args = [a for a in sys.argv[1:] if a != "--dry-run"] + dry_run = len(args) != len(sys.argv) - 1 + config_file = args[0] if args else "winpython_builds.toml" builds, python_versions = load_builds(config_file) - for build in builds: - run_build(build, python_versions) + for build in select_builds(builds, args[1:]): + run_build(build, python_versions, dry_run) if __name__ == "__main__": main() diff --git a/generate_a_winpython_distropy_meta.bat b/generate_a_winpython_distropy_meta.bat index c13a2bf4..6cfb9b0f 100644 --- a/generate_a_winpython_distropy_meta.bat +++ b/generate_a_winpython_distropy_meta.bat @@ -1,2 +1,3 @@ cd /D %~dp0 -call "C:\WinPdev\WPy64-31190\python-3.11.9.amd64\python.exe" %~dp0\build_winpython_meta.py %1 +rem [315 | 315:slim ...] [--dry-run] +call "C:\WinPdev\WPy64-31190\python-3.11.9.amd64\python.exe" %~dp0\build_winpython_meta.py %* diff --git a/winpython_builds.toml b/winpython_builds.toml new file mode 100644 index 00000000..19f54415 --- /dev/null +++ b/winpython_builds.toml @@ -0,0 +1,95 @@ +# WinPython local builds - every Python minor in one file. +# +# A build is a (python, flavor) pair. Everything else is derived: +# +# basedir = {root_dir_for_builds}\bd{python} +# requirements = {basedir}\{flavor requirements} +# source_dirs = {basedir}\{flavor source_dirs} +# wheelhousereq = {basedir}\{flavor wheelhousereq} +# toolsdirs = {root_dir_for_builds}\{flavor toolsdirs} +# +# so a new Python minor is one [pythons."3XX"] block, and its flavors are the +# names listed in "builds". +# +# Requirement files may differ per Python: name them again in +# [pythons."3XX".overrides.], which replaces that flavor's entries for +# that Python alone. Any other build key goes there too (bytecode, ...). +# +# Typical local build: +# generate_a_winpython_distropy_meta.bat winpython_builds.toml 315:slim +# generate_a_winpython_distropy_meta.bat winpython_builds.toml 315 +# generate_a_winpython_distropy_meta.bat winpython_builds.toml +# ... --dry-run prints the build commands instead of running them + +[defaults] +root_dir_for_builds = "C:\\Winp" +find_links = "C:\\Winp\\packages.srcreq" +arch = "64" +create_installer = "None" +# mandatory_requirements.txt and constraints.txt come from beside this file +# unless a build names its own. + +# ---- flavors: what makes a slim a slim -------------------------------------- + +[flavors.dot] +requirements = "dot_requirements.txt" +source_dirs = "packages.win-amd64" +toolsdirs = "bdTools\\Tools.dot" + +[flavors.dotf] +requirements = "dot_requirements.txt" +source_dirs = "packages.win-amd64t" +toolsdirs = "bdTools\\Tools.dot" + +[flavors.slim] +requirements = "requirements_slim.txt" +source_dirs = "packages.win-amd64" +toolsdirs = "bdTools\\tools64_pandoc_alone" + +[flavors.slimf] +requirements = "requirements_slimf.txt" +source_dirs = "packages.win-amd64t" +toolsdirs = "bdTools\\tools64_pandoc_alone" + +# Not built at the moment: no python below lists it. +[flavors.whl] +requirements = "dot_requirements.txt" +source_dirs = "packages.win-amd64" +toolsdirs = "bdTools\\Tools.dot" +wheelhousereq = "requirements_whl.txt" + +# ---- pythons: what gets built ----------------------------------------------- + +[pythons."313"] +python_target_release = "31315" +release = "0" +my_release_level = "b3" +builds = ["dot", "slim"] + +# A Python that names a requirements file its own way says so here: +# [pythons."313".overrides.slim] +# requirements = "requirements64_slim.txt" + +[pythons."314"] +python_target_release = "3147" +release = "0" +my_release_level = "b3" +builds = ["dot", "dotf", "slim", "slimf"] + +[pythons."314".overrides.slimf] +bytecode = "none" + +# [pythons."314".overrides.slim] +# create_installer = ".7z-mx7" + +[pythons."315"] +python_target_release = "3150" +release = "4" +my_release_level = "b3" +builds = ["dot", "dotf", "slim", "slimf"] + +# [pythons."315".overrides.slim] +# bytecode = "none" + +# [pythons."315".overrides.slimf] +# bytecode = "none" diff --git a/winpython_builds_bd13.toml b/winpython_builds_bd13.toml deleted file mode 100644 index 8c3fc047..00000000 --- a/winpython_builds_bd13.toml +++ /dev/null @@ -1,45 +0,0 @@ -[pythons] -"313" = { python_target_release = "31315", release = "0", my_release_level = "b3" } - - -[[builds]] -name = "dot" -root_dir_for_builds = "C:\\Winp" -python_target = "313" -flavor = "dot" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd313\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd313\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" - - -[[builds]] -name = "slim" -root_dir_for_builds = "C:\\Winp" -python_target = "313" -flavor = "slim" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd313\\requirements64_slim.txt" -source_dirs = "C:\\Winp\\bd313\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\tools64_pandoc_alone" - - -[[builds-nomore]] -name = "whl" -root_dir_for_builds = "C:\\Winp" -python_target = "313" -flavor = "whl" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd313\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd313\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" -wheelhousereq = "C:\\Winp\\bd313\\requirements64_whl.txt" diff --git a/winpython_builds_bd14.toml b/winpython_builds_bd14.toml deleted file mode 100644 index 5b86d6c8..00000000 --- a/winpython_builds_bd14.toml +++ /dev/null @@ -1,78 +0,0 @@ -[pythons] -"314" = { python_target_release = "3147", release = "0", my_release_level = "b3" } - - - -[[builds]] -name = "dot" -root_dir_for_builds = "C:\\Winp" -python_target = "314" -flavor = "dot" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd314\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd314\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" - -[[builds]] -name = "dotf" -root_dir_for_builds = "C:\\Winp" -python_target = "314" -flavor = "dotf" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd314\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd314\\packages.win-amd64t" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" - - - -[[builds]] -name = "slim" -root_dir_for_builds = "C:\\Winp" -python_target = "314" -flavor = "slim" -arch = "64" -# bytecode = "none" -create_installer = "none" -# create_installer = ".7z-mx7" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd314\\requirements_slim.txt" -source_dirs = "C:\\Winp\\bd314\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\tools64_pandoc_alone" - - - -[[builds]] -name = "slimf" -root_dir_for_builds = "C:\\Winp" -python_target = "314" -flavor = "slimf" -arch = "64" -bytecode = "none" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd314\\requirements_slimf.txt" -source_dirs = "C:\\Winp\\bd314\\packages.win-amd64t" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\tools64_pandoc_alone" - - -[[builds-nomore]] -name = "whl" -root_dir_for_builds = "C:\\Winp" -python_target = "314" -flavor = "whl" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd314\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd314\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" -wheelhousereq = "C:\\Winp\\bd314\\requirements_whl.txt" diff --git a/winpython_builds_bd15.toml b/winpython_builds_bd15.toml deleted file mode 100644 index 1704bd14..00000000 --- a/winpython_builds_bd15.toml +++ /dev/null @@ -1,64 +0,0 @@ -[pythons] -"315" = { python_target_release = "3150", release = "4", my_release_level = "b3" } - - - -[[builds]] -name = "dot" -root_dir_for_builds = "C:\\Winp" -python_target = "315" -flavor = "dot" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd315\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd315\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" - - -[[builds]] -name = "dotf" -root_dir_for_builds = "C:\\Winp" -python_target = "315" -flavor = "dotf" -arch = "64" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd315\\dot_requirements.txt" -source_dirs = "C:\\Winp\\bd315\\packages.win-amd64t" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\Tools.dot" - - - -[[builds]] -name = "slim" -root_dir_for_builds = "C:\\Winp" -python_target = "315" -flavor = "slim" -arch = "64" -# bytecode = "none" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd315\\requirements_slim.txt" -source_dirs = "C:\\Winp\\bd315\\packages.win-amd64" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\tools64_pandoc_alone" - - - -[[builds]] -name = "slimf" -root_dir_for_builds = "C:\\Winp" -python_target = "315" -flavor = "slimf" -arch = "64" -# bytecode = "none" -create_installer = "None" -mandatory_requirements = "C:\\Winp\\mandatory_requirements.txt" -requirements = "C:\\Winp\\bd315\\requirements_slimf.txt" -source_dirs = "C:\\Winp\\bd315\\packages.win-amd64t" -find_links = "C:\\Winp\\packages.srcreq" -toolsdirs = "C:\\Winp\\bdTools\\tools64_pandoc_alone" -