521df0eda0
A recent change in pip wheel cache behavior had upstream pip indicating that we really should be using pip wheel instead. The reason we weren't using pip wheel appears to be that we wanted to infer what top level wheel to install via contents of a dir separate from our wheel output dir/wheel cache. Using pip wheel implies everything gets flattened into one location. We deal with this by having the build tool write all of the top level wheels we care about into a separate location. Later we can install all of the top level wheels while pointing find links at the larger set of deps in the dir created by pip wheel. Change-Id: Id9c674c1ec6fe5e72534549082e3adda9e286fd5
125 lines
4.5 KiB
Bash
Executable File
125 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Copyright (c) 2019 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Make a list of bindep dependencies and a collection of built binary
|
|
# wheels for the repo in question as well as its python dependencies.
|
|
# Install javascript tools as well to support python that needs javascript
|
|
# at build time.
|
|
set -ex
|
|
|
|
mkdir -p /output/bindep
|
|
mkdir -p /output/wheels
|
|
|
|
cd /tmp/src
|
|
|
|
apt-get update
|
|
|
|
function install_bindep {
|
|
# Protect from the bindep builder image use of the assemble script
|
|
# to produce a wheel. Note we append because we want all
|
|
# sibling packages in here too
|
|
if [ -f bindep.txt -o -f other-requirements.txt ] ; then
|
|
bindep -l newline >> /output/bindep/run.txt || true
|
|
compile_packages=$(bindep -b compile || true)
|
|
if [ ! -z "$compile_packages" ] ; then
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y ${compile_packages}
|
|
fi
|
|
fi
|
|
}
|
|
|
|
function install_wheels {
|
|
# Build a wheel so that we have an install target.
|
|
# pip install . in the container context with the mounted
|
|
# source dir gets ... exciting.
|
|
# The `build` tool builds an sdist first then a wheel from
|
|
# that sdist which is exactly what we want. This triggers code
|
|
# generation steps such as are found in zuul, since the sequencing
|
|
# otherwise happens in a way that makes wheel content copying unhappy.
|
|
#
|
|
# This is a separate step before we use pip wheel so that we can write
|
|
# the top level wheels to install into a different location than all of
|
|
# our dependencies.
|
|
/tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./
|
|
|
|
# Pip wheel everything to build a collection of all wheels for all
|
|
# transitive depends. This ensures that the final install on the prod
|
|
# images can avoid installing any build depends. If a requirements.txt
|
|
# file exists, start there so that people can use git url syntax to do
|
|
# things like pick up patched but unreleased versions of dependencies.
|
|
# Only do this for the main package (i.e. only write requirements
|
|
# once).
|
|
if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then
|
|
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt
|
|
cp /tmp/src/requirements.txt /output/requirements.txt
|
|
fi
|
|
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl
|
|
|
|
# Pip wheel each of the extras so that we collect all possibly
|
|
# needed wheels in the wheel output dir. get-extras-packages also
|
|
# writes out the req files into /output/$extra/requirements.txt.
|
|
for req in $(get-extras-packages) ; do
|
|
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req"
|
|
done
|
|
}
|
|
|
|
PACKAGES=$*
|
|
|
|
# bindep the main package
|
|
install_bindep
|
|
|
|
# go through ZUUL_SIBLINGS, if any, and build those wheels too
|
|
for sibling in ${ZUUL_SIBLINGS:-}; do
|
|
pushd .zuul-siblings/${sibling}
|
|
install_bindep
|
|
popd
|
|
done
|
|
|
|
# Use a clean virtualenv for install steps to prevent things from the
|
|
# current environment making us not build a wheel.
|
|
python3 -m venv /tmp/venv
|
|
/tmp/venv/bin/pip install -U pip wheel build
|
|
|
|
# If there is an upper-constraints.txt file in the source tree,
|
|
# use it in the pip commands.
|
|
if [ -f /tmp/src/upper-constraints.txt ] ; then
|
|
cp /tmp/src/upper-constraints.txt /output/upper-constraints.txt
|
|
CONSTRAINTS="-c /tmp/src/upper-constraints.txt"
|
|
fi
|
|
|
|
# If we got a list of packages, get wheels for them, otherwise do
|
|
# this for the main package.
|
|
if [[ $PACKAGES ]] ; then
|
|
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES
|
|
for package in $PACKAGES ; do
|
|
echo "$package" >> /output/packages.txt
|
|
done
|
|
else
|
|
# pbr needs git installed, else nothing will work. Do this in between
|
|
# bindep and wheel so that we don't miss git in target images.
|
|
apt-get install -y git
|
|
|
|
install_wheels
|
|
fi
|
|
|
|
# Go through ZUUL_SIBLINGS, if any, and build those wheels too
|
|
for sibling in ${ZUUL_SIBLINGS:-}; do
|
|
pushd .zuul-siblings/${sibling}
|
|
install_wheels
|
|
popd
|
|
done
|
|
|
|
rm -rf /tmp/venv
|