# build_winpython.py import os, sys, argparse, datetime, subprocess, shutil import logging from pathlib import Path from filecmp import cmp def setup_logging(log_file: Path, LOG_FORMAT="%(asctime)s %(levelname)s: %(message)s"): """Initialize logging to both file and stdout.""" logging.basicConfig( level=logging.INFO, format=LOG_FORMAT, handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler(str(log_file), encoding="utf-8", mode='a') ] ) def log_section(message: str, highlight_bar="-"*40): logging.info("\n" + highlight_bar) logging.info(message) logging.info(highlight_bar) def delete_folder_if_exists(folder: Path, check_flavor: str = ""): check_last = folder.parent.name if not folder.is_dir() else folder.name expected_name = "bu" + check_flavor if folder.exists() and folder.is_dir() and check_last == expected_name: logging.info(f"Removing old backup: {folder}") folder_old = folder.with_suffix('.old') if folder_old.exists(): shutil.rmtree(folder_old) folder.rename(folder_old) shutil.rmtree(folder_old) def run_command(cmd, shell=False, check=True): logging.info(f"[RUNNING] {' '.join(cmd) if isinstance(cmd, list) else cmd}") with subprocess.Popen( cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True ) as proc: for line in proc.stdout: logging.info(line.rstrip()) if check and proc.wait() != 0: raise subprocess.CalledProcessError(proc.returncode, cmd) def pip_install(python_exe: Path, req_file: str, constraints: str, find_links: str, label: str, options:[]): if req_file and Path(req_file).exists(): cmd = [ str(python_exe), "-m", "pip", "install", "-r", req_file, "-c", constraints, "--pre", "--no-index", f"--find-links={find_links}", "--upgrade", "--no-warn-script-location" ] if options: cmd += options log_section(f"Pip-install {label}") run_command(cmd) else: log_section(f"No {label} specified/skipped") PARALLEL_DEFAULT_MAX = 4 # bare "parallel" never goes above this def parse_bytecode_mode(value: str): """Interpret --bytecode. Returns (skip pip's inline compile, compileall jobs). pip pip byte-compiles inline as it always has (default, unchanged) none no .pyc at all -- for throwaway builds: pylock, size pre-check parallel pip --no-compile, then one compileall pass over min(cpu_count, PARALLEL_DEFAULT_MAX) workers parallel-N same, with exactly N workers and no cap os.cpu_count() reports logical CPUs, so on a 4-core/8-thread laptop an uncapped default would start 8 workers for a job that measured I/O-bound rather than CPU-bound. Cap the default and let -N override deliberately. """ mode = (value or "pip").strip().lower() if mode == "pip": return False, None if mode == "none": return True, None if mode == "parallel": return True, min(os.cpu_count() or 1, PARALLEL_DEFAULT_MAX) if mode.startswith("parallel-") and mode[9:].isdigit() and int(mode[9:]) > 0: return True, int(mode[9:]) raise ValueError(f"--bytecode must be pip, none, parallel or parallel-N (got {value!r})") def compile_bytecode(target_python: Path, jobs: int): """Byte-compile site-packages in one pass, using the *target* interpreter. pip compiles serially while installing; doing it afterwards lets it run across cores. Failures stay non-fatal: some packages ship modules that are not importable on this Python, and pip tolerates those too. -W ignore matches what pip does. pip wraps its own byte-compilation in warnings.filterwarnings("ignore") (see pip/_internal/operations/install/ wheel.py), so the SyntaxWarnings that plenty of packages carry -- invalid escape sequences, `is` against a literal -- never reach the build log. Without this, moving the compile out of pip would surface all of them. """ site_packages = target_python.parent / "Lib" / "site-packages" log_section(f"Byte-compiling {site_packages} (-j{jobs})") run_command([str(target_python), "-W", "ignore", "-m", "compileall", "-q", f"-j{jobs}", str(site_packages)], check=False) def patch_winpython(python_exe): cmd = [ str(python_exe), "-c", "from wppm import wppm; wppm.Distribution().patch_standard_packages('', to_movable=True)" ] run_command(cmd) def check_env_bat(winpydirbase: Path): envbat = winpydirbase / "scripts" / "env.bat" if not envbat.exists(): raise FileNotFoundError(f"Missing env.bat at {envbat}") def generate_lockfiles(target_python: Path, winpydirbase: Path, constraints: str, find_links: str, file_postfix: str): pip_req = winpydirbase.parent / "requirement_temp.txt" with subprocess.Popen([str(target_python), "-m", "pip", "freeze"], stdout=subprocess.PIPE) as proc: packages = [l for l in proc.stdout if b"winpython" not in l] pip_req.write_bytes(b"".join(packages)) # Lock to web and local (scaffolding) for kind in ("", "local"): out = winpydirbase.parent / f"pylock.{file_postfix}{kind}.toml" outreq = winpydirbase.parent / f"requir.{file_postfix}{kind}.txt" cmd = [str(target_python), "-m", "pip", "lock", "--no-deps", "-c", constraints] if kind == "local": cmd += ["--find-links", find_links] cmd += ["-r", str(pip_req), "-o", str(out)] run_command(cmd) # Convert both locks to requirement.txt with hash256 cmd = [str(target_python), "-X", "utf8", "-c", f"from wppm import wheelhouse as wh; wh.pylock_to_req(r'{out}', r'{outreq}')"] run_command(cmd) # check equality web, local = "", "local" web_file = winpydirbase.parent / f"requir.{file_postfix}{web}.txt" local_file = winpydirbase.parent / f"requir.{file_postfix}{local}.txt" if not cmp(web_file, local_file): logging.info( f"⚠️⚠️⚠️⚠️⚠️⚠️ ALARM ⚠️⚠️⚠️⚠️⚠️⚠️ differences in {web_file} {local_file}") raise os.error else: logging.info(f"💖💖💖 match 💖💖💖 ok {web_file} {local_file}") logging.info(f"💖 housekeeping, removing local files 💖 {local_file}") os.remove (winpydirbase.parent / f"requir.{file_postfix}{local}.txt") os.remove (winpydirbase.parent / f"pylock.{file_postfix}{local}.toml") os.remove (winpydirbase.parent / f"requirement_temp.txt") if (winpydirbase.parent / f"requir.{file_postfix}_wheelslocal.txt").is_file(): os.remove (winpydirbase.parent / f"requir.{file_postfix}_wheelslocal.txt") if (winpydirbase.parent / f"pylock.{file_postfix}_wheelslocal.toml").is_file(): os.remove (winpydirbase.parent / f"pylock.{file_postfix}_wheelslocal.toml") # --- Main Logic --- def run_make_py(build_python, winpydirbase, args, winpyver, winpyver2): from . import make make.make_all( args.release, args.release_level, basedir_wpy=winpydirbase, verbose=True, flavor=args.flavor, source_dirs=args.source_dirs, toolsdirs=args.tools_dirs, winpyver=winpyver, winpyver2=winpyver2 ) def process_wheelhouse_requirements(target_python: Path, winpydirbase: Path,args: argparse.Namespace,file_postfix: str): """ Handle installation and conversion of wheelhouse requirements. """ wheelhousereq = Path(args.wheelhousereq) kind = "local" out = winpydirbase.parent / f"pylock.{file_postfix}_wheels{kind}.toml" outreq = winpydirbase.parent / f"requir.{file_postfix}_wheels{kind}.txt" if wheelhousereq.is_file(): # Generate pylock from wheelhousereq cmd = [ str(target_python), "-m", "pip", "lock", "--no-index", "--trusted-host=None", "--pre", "--find-links", args.find_links, "-c", args.constraints, "-r", str(wheelhousereq), "-o", str(out) ] run_command(cmd) # Convert pylock to requirements with hash pylock_to_req_cmd = [ str(target_python), "-X", "utf8", "-c", f"from wppm import wheelhouse as wh; wh.pylock_to_req(r'{out}', r'{outreq}')" ] run_command(pylock_to_req_cmd, check=False) kind = "" outw = winpydirbase.parent / f"pylock.{file_postfix}_wheels{kind}.toml" outreqw = winpydirbase.parent / f"requir.{file_postfix}_wheels{kind}.txt" # Generate web pylock from local frozen hashes web_lock_cmd = [ str(target_python), "-m", "pip", "lock", "--no-deps", "--require-hashes", "-r", str(outreq), "-o", str(outw) ] run_command(web_lock_cmd) pylock_to_req_cmd2 = [ str(target_python), "-X", "utf8", "-c", f"from wppm import wheelhouse as wh; wh.pylock_to_req(r'{outw}', r'{outreqw}')" ] run_command(pylock_to_req_cmd2, check=False) # Use wppm to download local from req made with web hashes wheelhouse = winpydirbase / "wheelhouse" / "included.wheels" wppm_cmd = [ str(target_python), "-X", "utf8", "-m", "wppm", str(out), "-ws", args.find_links, "-wd", str(wheelhouse) ] run_command(wppm_cmd, check=False) def main(): parser = argparse.ArgumentParser() parser.add_argument('--python-target', required=True, help='Target Python version, e.g. 311') parser.add_argument('--release', default='', help='Release') parser.add_argument('--flavor', default='', help='Build flavor') parser.add_argument('--arch', default='64', help='Architecture') parser.add_argument('--release-level', default='', help='Release level (e.g., b1, b2)') parser.add_argument('--winpydirbase', required=True, help='Path to put environment') parser.add_argument('--source_dirs', required=True, help='Path to directory with python zip') parser.add_argument('--tools_dirs', required=True, help='Path to directory with python zip') parser.add_argument('--buildenv', required=True, help='Path to build environment') parser.add_argument('--constraints', default='constraints.txt', help='Constraints file') parser.add_argument('--requirements', help='Main requirements.txt file') parser.add_argument('--find-links', default='wheelhouse', help='Path to local wheelhouse') parser.add_argument('--log-dir', default='WinPython_build_logs', help='Directory for logs') parser.add_argument('--mandatory-req', help='Mandatory requirements file') parser.add_argument('--wheelhousereq', help='Wheelhouse requirements file') parser.add_argument('--bytecode', default='pip', help="byte-compilation: 'pip' (inline, the default and what ships), " "'none' (no .pyc -- for throwaway builds such as pylock generation " "or a size pre-check), 'parallel' (pip --no-compile, then one " f"compileall pass over min(cpu_count, {PARALLEL_DEFAULT_MAX}) workers), " "or 'parallel-N' for exactly N workers. cpu_count is logical CPUs, " "so on a 4-core/8-thread machine 'parallel' uses 4, not 8") parser.add_argument('--create-installer', default='', help='default installer to create') args = parser.parse_args() # compute paths (same as Step2)... build_python = Path(args.buildenv) / "python.exe" winpydirbase = Path(args.winpydirbase) target_python = winpydirbase / "python" / "python.exe" # Setup paths and logs now = datetime.datetime.now() log_dir = Path(args.log_dir) log_dir.mkdir(exist_ok=True) time_str = now.strftime("%Y-%m-%d_at_%H%M") log_file = log_dir / f"build_{args.python_target}_{args.flavor}_{args.release_level}_{time_str}.txt" setup_logging(log_file) # Logs termination and version naming if len(args.release_level) > 0: z = Path(winpydirbase).name[(4+len(args.arch)):-len(args.release_level)-len(args.release)] else: z = Path(winpydirbase).name[(4+len(args.arch)):-len(args.release)] tada = f"{z[:1]}_{z[1:3]}_{z[3:]}_{args.release}" winpyver2 = tada.replace('_', '.') winpyver = f"{winpyver2}{args.flavor}{args.release_level}" file_postfix = f"{args.arch}-{tada}{args.flavor}{args.release_level}" log_section(f"Preparing build for Python {args.python_target} ({args.arch}-bit)") log_section(f"🙏 Step 1: displace old {Path(winpydirbase)}") delete_folder_if_exists(winpydirbase.parent, check_flavor=args.flavor) #bu{flavor]} log_section(f"🙏 Step 2: make.py Python with {str(build_python)} at ({winpydirbase}") run_make_py(str(build_python), winpydirbase, args, winpyver, winpyver2) check_env_bat(winpydirbase) log_section("🙏 Step 3: install requirements") no_compile, compile_jobs = parse_bytecode_mode(args.bytecode) pip_options = ["--force-reinstall"] + (["--no-compile"] if no_compile else []) for label, req in [ ("Mandatory", args.mandatory_req), ("Main", args.requirements), ]: pip_install(target_python, req, args.constraints, args.find_links, label, pip_options) log_section("🙏 Step 4: Patch Winpython") patch_winpython(target_python) # after patching, so the patched sources are the ones compiled if compile_jobs is not None: log_section("🙏 Step 4b: byte-compile") compile_bytecode(target_python, compile_jobs) elif no_compile: log_section("🙏 Step 4b: byte-compile skipped (--bytecode none)") log_section(f"🙏 Step 5: install wheelhouse requirements {args.wheelhousereq}") if args.wheelhousereq: process_wheelhouse_requirements(target_python, winpydirbase, args, file_postfix) log_section("🙏 Step 6: install lockfiles") print(target_python, winpydirbase, args.constraints, args.find_links, file_postfix) generate_lockfiles(target_python, winpydirbase, args.constraints, args.find_links, file_postfix) log_section(f"🙏 Step 7: generate changelog") mdn = f"WinPython{args.flavor}-{args.arch}bit-{winpyver2}.md" out = f"WinPython{args.flavor}-{args.arch}bit-{winpyver2}_History.md" changelog_dir = log_dir.parent/ "changelogs" cmd = ["set", f"WINPYVER2={winpyver2}&", "set", f"WINPYFLAVOR={args.flavor}&", "set", f"WINPYVER={winpyver2}{args.flavor}{args.release_level}&", str(target_python), "-X", "utf8", "-c" , ( "from wppm import wppm;" "result = wppm.Distribution().generate_package_index_markdown();" f"open(r'{winpydirbase.parent / mdn}', 'w', encoding='utf-8').write(result)" )] run_command(cmd, shell=True) shutil.copyfile (winpydirbase.parent / mdn, changelog_dir / mdn) cmd = [str(target_python), "-X", "utf8", "-c", ( "from wppm import diff;" f"result = diff.compare_package_indexes('{winpyver2}', searchdir=r'{changelog_dir}', flavor=r'{args.flavor}', architecture={args.arch});" f"open(r'{winpydirbase.parent / out}', 'w', encoding='utf-8').write(result)" )] run_command(cmd, check=False) shutil.copyfile (winpydirbase.parent / out, changelog_dir / out) if args.create_installer != "": log_section("🙏 Step 8: Create Installer") stem = f"WinPython{args.arch}-{winpyver2}{args.flavor}{args.release_level}" cmd = [str(target_python), "-c", f"from wppm import utils; utils.command_installer_7zip(r'{winpydirbase}', r'{winpydirbase.parent}', r'{stem}', r'{args.create_installer}')" ] run_command(cmd, check=False) if __name__ == '__main__': main()