From 74fd2b6a08d7b224dd20f90078abba2e4fe05c4a Mon Sep 17 00:00:00 2001 From: Davlet Panech Date: Tue, 13 Sep 2022 16:59:49 +0000 Subject: [PATCH] build-wheels: README file + minor script fixes Add a textfile describing the wheels cfg file format. Allow tarballs that pack their contents at top level. Story: 2010294 Task: 46279 Signed-off-by: Davlet Panech Change-Id: Ia28ca95d1be1a81e8481709139fc440f336de7f7 --- .../build-wheels/centos/dev-wheels-py2.cfg | 2 + .../build-wheels/centos/dev-wheels.cfg | 2 + .../build-wheels/centos/stable-wheels-py2.cfg | 2 + .../build-wheels/centos/stable-wheels.cfg | 2 + .../build-wheels/debian/stable-wheels-py2.cfg | 2 + .../build-wheels/debian/stable-wheels.cfg | 2 + build-tools/build-wheels/doc/wheels-cfg.md | 65 +++++++++++++++++++ .../docker-common/docker-build-wheel.sh | 15 +++-- 8 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 build-tools/build-wheels/doc/wheels-cfg.md diff --git a/build-tools/build-wheels/centos/dev-wheels-py2.cfg b/build-tools/build-wheels/centos/dev-wheels-py2.cfg index 244178d8..86ad94da 100644 --- a/build-tools/build-wheels/centos/dev-wheels-py2.cfg +++ b/build-tools/build-wheels/centos/dev-wheels-py2.cfg @@ -6,6 +6,8 @@ # # If fix_setup must be called, add |fix_setup at the end of the line # +# See doc/wheels-cfg.md for more info. +# amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 lz4-0.9.0-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0 diff --git a/build-tools/build-wheels/centos/dev-wheels.cfg b/build-tools/build-wheels/centos/dev-wheels.cfg index 2e0d7dad..f4c70d18 100644 --- a/build-tools/build-wheels/centos/dev-wheels.cfg +++ b/build-tools/build-wheels/centos/dev-wheels.cfg @@ -6,6 +6,8 @@ # # If fix_setup must be called, add |fix_setup at the end of the line # +# See doc/wheels-cfg.md for more info. +# amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl diff --git a/build-tools/build-wheels/centos/stable-wheels-py2.cfg b/build-tools/build-wheels/centos/stable-wheels-py2.cfg index 47f30a38..14f3f9e9 100644 --- a/build-tools/build-wheels/centos/stable-wheels-py2.cfg +++ b/build-tools/build-wheels/centos/stable-wheels-py2.cfg @@ -6,6 +6,8 @@ # # If fix_setup must be called, add |fix_setup at the end of the line # +# See doc/wheels-cfg.md for more info. +# abclient-0.2.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3 alembic-1.1.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9a/0f/a5e8997d58882da8ecd288360dddf133a83145de6480216774923b393422/alembic-1.1.0.tar.gz|alembic-1.1.0 amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 diff --git a/build-tools/build-wheels/centos/stable-wheels.cfg b/build-tools/build-wheels/centos/stable-wheels.cfg index fbe83628..d0b60395 100644 --- a/build-tools/build-wheels/centos/stable-wheels.cfg +++ b/build-tools/build-wheels/centos/stable-wheels.cfg @@ -6,6 +6,8 @@ # # If fix_setup must be called, add |fix_setup at the end of the line # +# See doc/wheels-cfg.md for more info. +# abclient-0.2.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3 alembic-1.4.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz|alembic-1.4.2 amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 diff --git a/build-tools/build-wheels/debian/stable-wheels-py2.cfg b/build-tools/build-wheels/debian/stable-wheels-py2.cfg index 57bf576b..9e332340 100644 --- a/build-tools/build-wheels/debian/stable-wheels-py2.cfg +++ b/build-tools/build-wheels/debian/stable-wheels-py2.cfg @@ -6,6 +6,8 @@ # # If fix_setup must be called, add |fix_setup at the end of the line # +# See doc/wheels-cfg.md for more info. +# abclient-0.2.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3 alembic-1.1.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9a/0f/a5e8997d58882da8ecd288360dddf133a83145de6480216774923b393422/alembic-1.1.0.tar.gz|alembic-1.1.0 amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 diff --git a/build-tools/build-wheels/debian/stable-wheels.cfg b/build-tools/build-wheels/debian/stable-wheels.cfg index 47d0571d..c1e8c391 100644 --- a/build-tools/build-wheels/debian/stable-wheels.cfg +++ b/build-tools/build-wheels/debian/stable-wheels.cfg @@ -6,6 +6,8 @@ # # If fix_setup must be called, add |fix_setup at the end of the line # +# See doc/wheels-cfg.md for more info. +# abclient-0.2.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3 alembic-1.4.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz|alembic-1.4.2 amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 diff --git a/build-tools/build-wheels/doc/wheels-cfg.md b/build-tools/build-wheels/doc/wheels-cfg.md new file mode 100644 index 00000000..f7fa5d2d --- /dev/null +++ b/build-tools/build-wheels/doc/wheels-cfg.md @@ -0,0 +1,65 @@ +# Syntax of wheels config files + +The files {debian,centos}/{stable,dev}-wheels.cfg list the 3rd-party wheels +(ie compiled python modules) to be included in the wheels tarball. Wheels are +listed one per line, each with the following "|"-separated fields. + +The first field is the wheel file name and is often python version & ABI +specific, eg: `lz4-0.9.0-cp39-cp39-linux_x86_64.whl`. + +The second field is `git`, `tar`, `zip`, or `pypi` and determines how to build +or download the wheel + +The third field is the URL of wheel source of the wheel itself + +The optional last field may be set to `fix_setup`, which will update +older/legacy module sources to use setuptools.setup, which is necessary for +building the module into a wheel. See script +`docker-common/docker-build-wheel.sh` for details. + +The exact number of fields depend on the wheel source: + +* git: `wheelname|git|repo-url|basedir|branch|fix_setup` + + Remote git repo URL + branch + + Example: + + ``` + lz4-0.9.0-cp39-cp39-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 + ``` + +* tar: `wheelname|tar|url|basedir|fix_setup` + + Source tarball URL. Basedir must be the subdirectory containing the package + source code within the tarball, and typically equals the basename of the + tarfile and/or module name & version. If tarfile contains source files at + top level, this parameter must be empty or "." + + Example: + + ``` + abclient-0.2.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3i + ``` + +* zip: `wheelname|zip|url|basedir|fix_setup` + + Same as `tar`, but in .zip format + + Example: + + ``` + networkx-2.2-py2.py3-none-any.whl|zip|https://files.pythonhosted.org/packages/f3/f4/7e20ef40b118478191cec0b58c3192f822cace858c19505c7670961b76b2/networkx-2.2.zip|networkx-2.2 + ``` + +* pypi: `wheelname|pypi|url` + + URL of a pre-built wheel for direct download. This type of download avoids + the build step and reduces overall build times. + + Example: + + ``` + bottle-0.12.18-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/e9/39/2bf3a1fd963e749cdbe5036a184eda8c37d8af25d1297d94b8b7aeec17c4/bottle-0.12.18-py3-none-any.whl + ``` + diff --git a/build-tools/build-wheels/docker-common/docker-build-wheel.sh b/build-tools/build-wheels/docker-common/docker-build-wheel.sh index 59e752a4..fbdde368 100755 --- a/build-tools/build-wheels/docker-common/docker-build-wheel.sh +++ b/build-tools/build-wheels/docker-common/docker-build-wheel.sh @@ -218,11 +218,11 @@ function from_tar { tarball=$(basename $wgetsrc) if [[ $tarball =~ gz$ ]]; then - taropts="xzf" + taropts="-xzf" elif [[ $tarball =~ bz2$ ]]; then - taropts="xjf" + taropts="-xjf" else - taropts="xf" + taropts="-xf" fi with_retries ${MAX_ATTEMPTS} wget $wgetsrc @@ -231,7 +231,14 @@ function from_tar { continue fi - tar $taropts $(basename $wgetsrc) + local -a tar_base_opts + if [[ -z "$basedir" || "$basedir" == "." ]] ; then + basedir="$(echo "$tarball" | sed -r -e 's#[.]tar([.].*)?$##')" + mkdir -p "$basedir" + tar_basedir_opts+=("-C" "$basedir") + fi + + tar "${tar_basedir_opts[@]}" $taropts $(basename $wgetsrc) if [ $? -ne 0 ]; then echo "Failure running: tar $taropts $(basename $wgetsrc)" echo $wheelname >> $FAILED_LOG