From 4dc8023639a43f1d28811d4e504c0a7a52762891 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 15 Dec 2024 01:15:06 +0000 Subject: [PATCH] build: sync with upstream 2024-12-15 https://github.com/void-linux/void-packages/commit/01a57d87 --- common/build-helper/python3.sh | 2 + .../environment/build-style/python3-module.sh | 3 + .../environment/build-style/python3-pep517.sh | 5 +- common/environment/setup-subpkg/subpkg.sh | 6 + common/environment/setup/python.sh | 2 +- common/hooks/do-pkg/00-gen-pkg.sh | 4 +- .../04-create-xbps-metadata-scripts.sh | 2 +- .../hooks/pre-configure/02-script-wrapper.sh | 2 +- common/hooks/pre-pkg/04-generate-provides.sh | 29 +++ common/hooks/pre-pkg/06-verify-python-deps.sh | 17 ++ common/scripts/parse-py-metadata.py | 203 ++++++++++++++++++ common/shlibs | 18 +- xbps-src | 2 +- 13 files changed, 280 insertions(+), 15 deletions(-) create mode 100644 common/hooks/pre-pkg/04-generate-provides.sh create mode 100644 common/hooks/pre-pkg/06-verify-python-deps.sh create mode 100644 common/scripts/parse-py-metadata.py diff --git a/common/build-helper/python3.sh b/common/build-helper/python3.sh index 4707599..1a38f32 100644 --- a/common/build-helper/python3.sh +++ b/common/build-helper/python3.sh @@ -2,8 +2,10 @@ if [ -n "$CROSS_BUILD" ]; then export PYPREFIX="$XBPS_CROSS_BASE" export CFLAGS+=" -I${XBPS_CROSS_BASE}/${py3_inc} -I${XBPS_CROSS_BASE}/usr/include" + export CXXFLAGS+=" -I${XBPS_CROSS_BASE}/${py3_inc} -I${XBPS_CROSS_BASE}/usr/include" export LDFLAGS+=" -L${XBPS_CROSS_BASE}/${py3_lib} -L${XBPS_CROSS_BASE}/usr/lib" export CC="${XBPS_CROSS_TRIPLET}-gcc -pthread $CFLAGS $LDFLAGS" + export CXX="${XBPS_CROSS_TRIPLET}-g++ -pthread $CXXFLAGS $LDFLAGS" export LDSHARED="${CC} -shared $LDFLAGS" export PYTHON_CONFIG="${XBPS_CROSS_BASE}/usr/bin/python3-config" export PYTHONPATH="${XBPS_CROSS_BASE}/${py3_lib}" diff --git a/common/environment/build-style/python3-module.sh b/common/environment/build-style/python3-module.sh index 638f6be..8ab040a 100644 --- a/common/environment/build-style/python3-module.sh +++ b/common/environment/build-style/python3-module.sh @@ -1,3 +1,6 @@ lib32disabled=yes +if [ -z "$nopyprovides" ] || [ -z "$noverifypydeps" ]; then + hostmakedepends+=" python3-packaging-bootstrap" +fi makedepends+=" python3" build_helper+=" python3" diff --git a/common/environment/build-style/python3-pep517.sh b/common/environment/build-style/python3-pep517.sh index f4faf98..b567737 100644 --- a/common/environment/build-style/python3-pep517.sh +++ b/common/environment/build-style/python3-pep517.sh @@ -1,3 +1,6 @@ -hostmakedepends+=" python3-build python3-installer" lib32disabled=yes +hostmakedepends+=" python3-build python3-installer" +if [ -z "$nopyprovides" ] || [ -z "$noverifypydeps" ]; then + hostmakedepends+=" python3-packaging-bootstrap" +fi build_helper+=" python3" diff --git a/common/environment/setup-subpkg/subpkg.sh b/common/environment/setup-subpkg/subpkg.sh index 6edab5d..379ade7 100644 --- a/common/environment/setup-subpkg/subpkg.sh +++ b/common/environment/setup-subpkg/subpkg.sh @@ -11,6 +11,9 @@ unset -v nostrip nostrip_files # hooks/post-install/14-fix-permissions unset -v nocheckperms nofixperms +# hooks/pre-pkg/04-generate-provides +unset -v nopyprovides + # hooks/pre-pkg/04-generate-runtime-deps unset -v noverifyrdeps skiprdeps allow_unknown_shlibs shlib_requires @@ -20,6 +23,9 @@ unset -v lib32depends lib32disabled lib32files lib32mode lib32symlinks # hooks/pre-pkg/06-shlib-provides unset -v noshlibprovides shlib_provides +# hooks/pre-pkg/06-verify-python-deps +unset -v noverifypydeps python_extras + # xbps-triggers: system-accounts unset -v system_accounts system_groups diff --git a/common/environment/setup/python.sh b/common/environment/setup/python.sh index ab5c47d..75b3399 100644 --- a/common/environment/setup/python.sh +++ b/common/environment/setup/python.sh @@ -7,7 +7,7 @@ py2_lib="usr/lib/python${py2_ver}" py2_sitelib="${py2_lib}/site-packages" py2_inc="usr/include/python${py2_ver}" -py3_ver="3.12" +py3_ver="3.13" py3_abiver="" py3_lib="usr/lib/python${py3_ver}" py3_sitelib="${py3_lib}/site-packages" diff --git a/common/hooks/do-pkg/00-gen-pkg.sh b/common/hooks/do-pkg/00-gen-pkg.sh index aa48214..06ff08b 100644 --- a/common/hooks/do-pkg/00-gen-pkg.sh +++ b/common/hooks/do-pkg/00-gen-pkg.sh @@ -40,6 +40,9 @@ genpkg() { if [ -s ${XBPS_STATEDIR}/${pkgname}${suffix}-shlib-provides ]; then _shprovides="$(<${XBPS_STATEDIR}/${pkgname}${suffix}-shlib-provides)" fi + if [ -s ${XBPS_STATEDIR}/${pkgname}${suffix}-provides ]; then + _provides="$(<${XBPS_STATEDIR}/${pkgname}${suffix}-provides)" + fi if [ -s ${XBPS_STATEDIR}/${pkgname}${suffix}-shlib-requires ]; then _shrequires="$(<${XBPS_STATEDIR}/${pkgname}${suffix}-shlib-requires)" fi @@ -48,7 +51,6 @@ genpkg() { fi # Stripping whitespaces - local _provides="$(echo $provides)" local _conflicts="$(echo $conflicts)" local _replaces="$(echo $replaces)" local _reverts="$(echo $reverts)" diff --git a/common/hooks/post-install/04-create-xbps-metadata-scripts.sh b/common/hooks/post-install/04-create-xbps-metadata-scripts.sh index 70152ba..cf7868b 100644 --- a/common/hooks/post-install/04-create-xbps-metadata-scripts.sh +++ b/common/hooks/post-install/04-create-xbps-metadata-scripts.sh @@ -274,7 +274,7 @@ _EOF if [ -d ${PKGDESTDIR}/usr/lib/python* ]; then pycompile_version="$(find ${PKGDESTDIR}/usr/lib/python* -prune -type d | grep -o '[[:digit:]]\.[[:digit:]]\+$')" if [ -z "${pycompile_module}" ]; then - pycompile_module="$(find ${PKGDESTDIR}/usr/lib/python*/site-packages -mindepth 1 -maxdepth 1 '!' -name '*.egg-info' '!' -name '*.dist-info' '!' -name '*.so' '!' -name '*.pth' -printf '%f ')" + pycompile_module="$(find ${PKGDESTDIR}/usr/lib/python*/site-packages* -mindepth 1 -maxdepth 1 '!' -name '*.egg-info' '!' -name '*.dist-info' '!' -name '*.so' '!' -name '*.pth' -printf '%f ')" fi fi diff --git a/common/hooks/pre-configure/02-script-wrapper.sh b/common/hooks/pre-configure/02-script-wrapper.sh index 02f3965..3d5d6c2 100644 --- a/common/hooks/pre-configure/02-script-wrapper.sh +++ b/common/hooks/pre-configure/02-script-wrapper.sh @@ -251,7 +251,7 @@ hook() { generic_wrapper3 libetpan-config generic_wrapper3 giblib-config python_wrapper python-config 2.7 - python_wrapper python3-config 3.12 + python_wrapper python3-config 3.13 apr_apu_wrapper apu-1-config qemu_wrapper llvm-config } diff --git a/common/hooks/pre-pkg/04-generate-provides.sh b/common/hooks/pre-pkg/04-generate-provides.sh new file mode 100644 index 0000000..f9c1a68 --- /dev/null +++ b/common/hooks/pre-pkg/04-generate-provides.sh @@ -0,0 +1,29 @@ +# vim: set ts=4 sw=4 et: +# +# This hook executes the following tasks: +# - Generates provides file with provides entries for xbps-create(1) + +generate_python_provides() { + local py3_bin="${XBPS_MASTERDIR}/usr/bin/python3" + + if [ -z "$nopyprovides" ] && [ -d "${PKGDESTDIR}/${py3_sitelib}" ] && [ -x "${py3_bin}" ]; then + PYTHONPATH="${XBPS_MASTERDIR}/${py3_sitelib}-bootstrap" "${py3_bin}" \ + "${XBPS_COMMONDIR}"/scripts/parse-py-metadata.py \ + -S "${PKGDESTDIR}/${py3_sitelib}" -v "${pkgver}" provides + fi +} + +hook() { + local -a _provides=() + + # include explicit values from the template + read -r -a _provides <<< "$provides" + + # get the canonical python package names for each python module + mapfile -t _provides -O "${#_provides[@]}" < <( generate_python_provides ) + + if [ "${#_provides[@]}" -gt 0 ]; then + echo " ${_provides[*]}" + echo "${_provides[*]}" > "${XBPS_STATEDIR}/${pkgname}-provides" + fi +} diff --git a/common/hooks/pre-pkg/06-verify-python-deps.sh b/common/hooks/pre-pkg/06-verify-python-deps.sh new file mode 100644 index 0000000..a47e5bb --- /dev/null +++ b/common/hooks/pre-pkg/06-verify-python-deps.sh @@ -0,0 +1,17 @@ +# vim: set ts=4 sw=4 et: +# +# This hook executes the following tasks: +# - Verifies python module dependencies from dist-info's METADATA and egg-info's PKG-INFO + +hook() { + local py3_bin="${XBPS_MASTERDIR}/usr/bin/python3" + + if [ -z "$nopyprovides" ] && [ -d "${PKGDESTDIR}/${py3_sitelib}" ] && [ -x "${py3_bin}" ]; then + PYTHONPATH="${XBPS_MASTERDIR}/${py3_sitelib}-bootstrap" "${py3_bin}" \ + "${XBPS_COMMONDIR}"/scripts/parse-py-metadata.py \ + ${NOCOLORS:+-C} ${XBPS_STRICT:+-s} -S "${PKGDESTDIR}/${py3_sitelib}" -v "${pkgver}" \ + depends -e "${python_extras}" \ + -V <( $XBPS_QUERY_XCMD -R -p provides -s "py3:" ) -D "${XBPS_STATEDIR}/${pkgname}-rdeps" \ + || msg_error "$pkgver: failed to verify python module dependencies\n" + fi +} diff --git a/common/scripts/parse-py-metadata.py b/common/scripts/parse-py-metadata.py new file mode 100644 index 0000000..150d476 --- /dev/null +++ b/common/scripts/parse-py-metadata.py @@ -0,0 +1,203 @@ +#!/usr/bin/python3 + +# vim: set ts=4 sw=4 et: +""" +Usage: + +./parse-py-metadata.py -S "$DESTDIR/$py3_sitelib" provides -v "$version" + + extract the names of top-level packages from: + - $DESTDIR/$py3_sitelib/*.dist-info/METADATA + - $DESTDIR/$py3_sitelib/*.egg-info/PKG-INFO + +./parse-py-metadata.py -S "$DESTDIR/$py3_sitelib" [-s] [-C] depends -e "extra1 extra2 ..." + -D "$XBPS_STATEDIR/$pkgname-rdeps" -V <( xbps-query -R -p provides -s "py3:" ) + + check that the dependencies of a package match what's listed in the python + package metadata, using the virtual package provides entries generated by + `parse-py-metadata.py provides`. + +This script requires python3-packaging-bootstrap to be installed in the chroot +to run (which should be taken care of by the python3-module and python3-pep517 +build styles). +""" + +import argparse +from pathlib import Path +from sys import stderr +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from packaging.metadata import Metadata + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + + +def msg_err(msg: str, *, nocolor: bool = False, strict: bool = False): + if nocolor: + print(msg, flush=True) + else: + color = "31" if strict else "33" + print(f"\033[1m\033[{color}m{msg}\033[m", file=stderr, flush=True) + + +def vpkgname(val: "str | Requirement", *, version: str | None = None) -> str: + sfx = "" + if version is not None: + sfx = f"-{version}" + if isinstance(val, Requirement): + name = val.name + else: + name = val + return f"py3:{canonicalize_name(name)}{sfx}" + + +def getpkgname(pkgver: str) -> str: + return pkgver.rpartition("-")[0] + + +def getpkgversion(pkgver: str) -> str: + return pkgver.rpartition("-")[2] + + +def getpkgdepname(pkgdep: str) -> str: + if "<" in pkgdep: + return pkgdep.partition("<")[0] + elif ">" in pkgdep: + return pkgdep.partition(">")[0] + else: + return pkgdep.rpartition("-")[0] + + +def match_markers(req: "Requirement", extras: set[str]) -> bool: + # unconditional requirement + if req.marker is None: + return True + + # check the requirement for each extra we want and without any extras + if extras: + return req.marker.evaluate() and any(req.marker.evaluate({"extra": e}) for e in extras) + + return req.marker.evaluate() + + +def find_metadata_files(sitepkgs: Path) -> list[Path]: + metafiles = list(sitepkgs.glob("*.dist-info/METADATA")) + metafiles.extend(sitepkgs.glob("*.egg-info/PKG-INFO")) + return metafiles + + +def parse_provides(args): + out = set() + + for metafile in find_metadata_files(args.sitepkgs): + with metafile.open() as f: + raw = f.read() + + meta = Metadata.from_email(raw, validate=False) + + out.add(vpkgname(meta.name, version=getpkgversion(args.pkgver))) + if meta.provides_dist is not None: + out.update(map(lambda n: vpkgname(n, version=getpkgversion(args.pkgver)), meta.provides_dist)) + # deprecated but may be used + if meta.provides is not None: + out.update(map(lambda n: vpkgname(n, version=getpkgversion(args.pkgver)), meta.provides)) + + print("\n".join(out), flush=True) + + +def parse_depends(args): + depends = dict() + vpkgs = dict() + extras = set(args.extras.split()) + + with args.vpkgs.open() as f: + for ln in f.readlines(): + if not ln.strip(): + continue + pkgver, _, rest = ln.partition(":") + vpkgvers, _, _ = rest.strip().partition("(") + pkg = getpkgname(pkgver) + vpkg = map(getpkgname, vpkgvers.split()) + for v in vpkg: + vpkgs[v] = pkg + + if args.rdeps.exists(): + with args.rdeps.open() as f: + rdeps = list(map(getpkgdepname, f.read().split())) + else: + rdeps = [] + + for metafile in find_metadata_files(args.sitepkgs): + with metafile.open() as f: + raw = f.read() + + meta = Metadata.from_email(raw, validate=False) + + if meta.requires_dist is not None: + depends.update(map(lambda p: (vpkgname(p), None), + filter(lambda r: match_markers(r, extras), meta.requires_dist))) + # deprecated but may be used + if meta.requires is not None: + depends.update(map(lambda p: (vpkgname(p), None), meta.requires)) + + err = False + unknown = False + missing = [] + for k in depends.keys(): + if k in vpkgs.keys(): + pkgname = vpkgs[k] + if pkgname in rdeps: + print(f" PYTHON: {k} <-> {pkgname}", flush=True) + else: + msg_err(f" PYTHON: {k} <-> {pkgname} NOT IN depends PLEASE FIX!", + nocolor=args.nocolor, strict=args.strict) + missing.append(pkgname) + err = True + else: + msg_err(f" PYTHON: {k} <-> UNKNOWN PKG PLEASE FIX!", + nocolor=args.nocolor, strict=args.strict) + unknown = True + err = True + + if missing or unknown: + msg_err(f"=> {args.pkgver}: missing dependencies detected!", + nocolor=args.nocolor, strict=args.strict) + if missing: + msg_err(f"=> {args.pkgver}: please add these packages to depends: {' '.join(sorted(missing))}", + nocolor=args.nocolor, strict=args.strict) + + if err and args.strict: + exit(1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-S", dest="sitepkgs", type=Path) + parser.add_argument("-v", dest="pkgver") + parser.add_argument("-s", dest="strict", action="store_true") + parser.add_argument("-C", dest="nocolor", action="store_true") + subparsers = parser.add_subparsers() + + prov_parser = subparsers.add_parser("provides") + prov_parser.set_defaults(func=parse_provides) + + deps_parser = subparsers.add_parser("depends") + deps_parser.add_argument("-e", dest="extras", default="") + deps_parser.add_argument("-V", dest="vpkgs", type=Path) + deps_parser.add_argument("-D", dest="rdeps", type=Path) + deps_parser.set_defaults(func=parse_depends) + + args = parser.parse_args() + + try: + from packaging.metadata import Metadata + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + except ImportError: + msg_err(f"=> WARNING: {args.pkgver}: missing packaging module!\n" + f"=> WARNING: {args.pkgver}: please add python3-packaging-bootstrap to hostmakedepends to run this check", + nocolor=args.nocolor) + exit(0) + + args.func(args) diff --git a/common/shlibs b/common/shlibs index 5e8f454..e2742f9 100644 --- a/common/shlibs +++ b/common/shlibs @@ -673,8 +673,8 @@ libboost_stacktrace_noop.so.1.83.0 libboost_stacktrace_noop-1.83.0_1 libboost_stacktrace_addr2line.so.1.83.0 libboost_stacktrace_addr2line-1.83.0_1 libboost_stacktrace_basic.so.1.83.0 libboost_stacktrace_basic-1.83.0_1 libboost_mpi.so.1.83.0 libboost_mpi-1.83.0_1 -libboost_mpi_python311.so.1.83.0 libboost_mpi-python3-1.83.0_1 -libboost_python312.so.1.83.0 boost-python3-1.83.0_1 +libboost_mpi_python313.so.1.83.0 libboost_mpi-python3-1.83.0_5 +libboost_python313.so.1.83.0 boost-python3-1.83.0_5 libexempi.so.8 exempi-2.5.0_1 libatasmart.so.4 libatasmart-0.17_1 libsgutils2-1.45.so.2 libsgutils-1.45_1 @@ -1730,8 +1730,8 @@ libportaudio.so.2 portaudio-19.20140130_1 libportaudiocpp.so.0 portaudio-cpp-19.20140130_1 libdar.so.6000 libdar-2.6.6_1 libdar64.so.6000 libdar-2.6.6_1 -libpython3.so python3-3.12.0_1 -libpython3.12.so.1.0 python3-3.12.0_1 +libpython3.so python3-3.13.0_1 +libpython3.13.so.1.0 python3-3.13.0_1 libbrscandec2.so.1 brother-brscan3-0.2.11_2 libxmlrpc_server_abyss.so.3 xmlrpc-c-1.51.06_2 libxmlrpc++.so.8 xmlrpc-c-1.51.06_2 @@ -2328,8 +2328,8 @@ libetpan.so.20 libetpan-1.9.3_4 libxmp.so.4 libxmp-4.3.7_1 libKF6ThreadWeaver.so.6 kf6-threadweaver-6.0.0_1 libKF5ThreadWeaver.so.5 threadweaver-5.26.0_1 -libOpenImageIO_Util.so.2.4 openimageio-2.4.9.0_1 -libOpenImageIO.so.2.4 openimageio-2.4.9.0_1 +libOpenImageIO_Util.so.2.5 openimageio-2.5.17.0_1 +libOpenImageIO.so.2.5 openimageio-2.5.17.0_1 libOpenColorIO.so.2.1 opencolorio-2.1.2_1 libpystring.so.1 pystring-1.1.3_1 libyaml-cpp.so.0.7 yaml-cpp-0.7.0_1 @@ -3980,13 +3980,13 @@ librdkafka.so.1 librdkafka-1.4.4_3 librdkafka++.so.1 librdkafka-1.4.4_3 libraft.so.0 raft-0.18.2_1 libmdnsd.so.1 libmdnsd-0.9_1 -libosdGPU.so.3.4.3 OpenSubdiv-3.4.3_1 -libosdCPU.so.3.4.3 OpenSubdiv-3.4.3_1 +libosdGPU.so.3.6.0 OpenSubdiv-3.6.0_1 +libosdCPU.so.3.6.0 OpenSubdiv-3.6.0_1 libcaribou.so.0 libcaribou-0.4.21_3 libtinyclipboard.so.1 tinyclipboard-16.01_1 libcbor.so.0.11 libcbor-0.11.0_1 libfido2.so.1 libfido2-1.6.0_2 -libOpenImageDenoise.so.1 openimagedenoise-1.3.0_1 +libOpenImageDenoise.so.2 openimagedenoise-2.3.0_1 libcbang0.so cbang-1.6.0_3 libblosc.so.1 c-blosc-1.17.1_1 libopenvdb.so.9.0 openvdb-9.0.0_1 diff --git a/xbps-src b/xbps-src index 00e955f..c17270b 100755 --- a/xbps-src +++ b/xbps-src @@ -244,7 +244,7 @@ $(print_cross_targets) This alternative repository will also be used to resolve dependencies with highest priority order than others. --s Make vsed warnings errors. +-s Make some warnings into errors. -t Create a temporary masterdir to not pollute the current one. Note that the existing masterdir must be fully populated with binary-bootstrap first.