353 lines
16 KiB
Bash
353 lines
16 KiB
Bash
#!/bin/sh
|
|
|
|
# --------------------------------------------------------------------------------
|
|
# Introduction
|
|
# --------------------------------------------------------------------------------
|
|
# To distribute Python with UE5 on Mac, it must built locally and the produced binaries install names must
|
|
# be updated to support relocation (so that the libraries are found whereever the user installs UE. We also
|
|
# need to build universal binaries (Intel x64 and ARM M1) and support various version of MacOS. This script
|
|
# is meant to help with that. Note that the binaries you downloaod from Python.org are not relocatable and the
|
|
# the binaries are signed, so we cannot fix that with install_name_tool because it breaks the signature.
|
|
# We really need to build that stuff.
|
|
#
|
|
# --------------------------------------------------------------------------------
|
|
# Steps
|
|
# --------------------------------------------------------------------------------
|
|
# - Download the desired Python version from https://www.python.org/ftp/python/
|
|
# - Download xz (liblzma) from https://tukaani.org/xz/
|
|
# - Unpack the downloaded packages somewhere. ex: $VolumeRoot/Build/
|
|
# - Copy OpenSSL from Engine/Source/ThirdParty/OpenSSL into $VolumeRoot/Deploy
|
|
# - Copy zlib from Engine/Source/ThirdParty/zlib into $VolumeRoot/Deploy
|
|
# - Build lzma for x64 architecture.
|
|
# cd $VolumeRoot/Build/xz-5.4.6
|
|
# ./configure --prefix=$VolumeRoot/Deploy/xz-5.4.6 --disable-xz --disable-lzma-links --disable-scripts --disable-doc CFLAGS="-isysroot `xcrun --sdk macosx --show-sdk-path` -mmacosx-version-min=11.0 -gdwarf-2 -arch x86_64" CPPFLAGS="-mmacosx-version-min=11.0 -gdwarf-2 -arch x86_64" LDFLAGS="-mmacosx-version-min=11.0 -arch x86_64"
|
|
# make -j28
|
|
# make install
|
|
# mv $VolumeRoot/Deploy/xz-5.4.6/lib $VolumeRoot/Deploy/xz-5.4.6/lib_x64
|
|
# - Build lzma for ARM architecture
|
|
# make clean
|
|
# ./configure --prefix=$VolumeRoot/Deploy/xz-5.4.6 --host=aarch64-apple-darwin --disable-xz --disable-lzma-links --disable-scripts --disable-doc CFLAGS="-isysroot `xcrun --sdk macosx --show-sdk-path` -mmacosx-version-min=11.0 -gdwarf-2 -arch arm64" CPPFLAGS="-mmacosx-version-min=11.0 -gdwarf-2 -arch arm64" LDFLAGS="-mmacosx-version-min=11.0 -arch arm64"
|
|
# make -j28
|
|
# make install
|
|
# mv $VolumeRoot/Deploy/xz-5.4.6/lib $VolumeRoot/Deploy/xz-5.4.6/lib_arm
|
|
# - Smash lzma x64 and ARM libraries together
|
|
# mkdir $VolumeRoot/Deploy/xz-5.4.6/lib
|
|
# lipo -create $VolumeRoot/Deploy/xz-5.4.6/lib_x64/liblzma.a $VolumeRoot/Deploy/xz-5.4.6/lib_arm/liblzma.a -output $VolumeRoot/Deploy/xz-5.4.6/lib/liblzma.a
|
|
# - Export the variables to make it easy (adjust the versions)
|
|
# export SSL_HOME=$VolumeRoot/Deploy/OpenSSL/1.1.1t/openssl
|
|
# export ZLIB_HOME=$VolumeRoot/Deploy/zlib/1.3
|
|
# export LZMA_HOME=$VolumeRoot/Deploy/xz-5.4.6
|
|
# - Go to the source directory '$VolumeRoot/Build/python-3.11.8' and run the configure command (might need some fixes if you updated any of the relative paths)
|
|
# ./configure --prefix=$VolumeRoot/Deploy/Python3.11.8 --enable-shared --enable-universalsdk=`xcrun --sdk macosx --show-sdk-path` --with-universal-archs=universal2 --enable-optimizations CFLAGS="-isysroot `xcrun --sdk macosx --show-sdk-path` -mmacosx-version-min=11.0 -gdwarf-2 -I$ZLIB_HOME/include -I$LZMA_HOME/include" CPPFLAGS="-mmacosx-version-min=11.0 -gdwarf-2 -I$ZLIB_HOME/include -I$LZMA_HOME/include" LDFLAGS="$ZLIB_HOME/lib/Mac/Release/libz.a $LZMA_HOME/lib/liblzma.a -mmacosx-version-min=11.0" --with-openssl=$SSL_HOME
|
|
# make -j28
|
|
# make install
|
|
# - Adjust the variables below according to your setup.
|
|
# - Run this scripts. It copies the binaries and fix their install name.
|
|
#
|
|
# To statically link OpenSSL, add the following to Modules/Setup.local in the python source location after running ./configure, but before make
|
|
#OPENSSL_INCLUDES=-I$(SSL_HOME)/include
|
|
#OPENSSL_LDFLAGS=-L$(SSL_HOME)/lib
|
|
# _ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \
|
|
# -lssl -Wl,-hidden-lssl \
|
|
# -lcrypto -Wl,-hidden-lcrypto
|
|
# _hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \
|
|
# -lcrypto -Wl,-hidden-lcrypto
|
|
|
|
|
|
|
|
#
|
|
# --------------------------------------------------------------------------------
|
|
# Install names
|
|
# --------------------------------------------------------------------------------
|
|
# On Mac, executables and dynamic libraries (.dylib, .so) contains install name ids to locate their dependencies. Because
|
|
# we copy, move and redistribute Python, the stored ids, which are paths, should be relative, so the end-user can put UE5 anywhere on
|
|
# his system. Unfortunately, the build system set hardcoded path. I didn't figure out how to pass the correct options to ./configure
|
|
# and/or make install to make them relative. So it is done by this script before copying in UE5 tree.
|
|
#
|
|
# To view a binary dependencies, we can use 'otool'
|
|
#
|
|
# otool -L python3.9 -> View the dynamic library install name ids loaded by the executable. (ids are paths)
|
|
# otool -l python3.9 -> View the LC_RPATH (rpath stored in the executable)
|
|
# otool -L libpython3.9.dylib -> View this dynamic library install name id and the dynamic librairy install name ids loaded by this libraries.
|
|
# otool -L hashlib.so -> View this dynamic library install name id and the dynamic librairy install name ids loaded by this libraries.
|
|
#
|
|
# To support relocating python elsewhere, we need to update the install name ids (the path) with tokens that are going to
|
|
# be pattern matched at load time. For executable like UnrealEditor or python3.9, we need to tell the dynamic loader where
|
|
# to look for the libraries, relative to the executable. An executable can have serveral "rpath" values. For python3.9
|
|
# executable, we want to add this one:
|
|
#
|
|
# install_name_tool -add_rpath @executable_path/../lib python3.9
|
|
#
|
|
# This gives one possible value to the token "@rpath" that we will used when searching the dependent librairies. Now
|
|
# we want to replace the libraries install name ids loaded by python3.9 executable to be relative too. This can be done as following:
|
|
#
|
|
# install_name_tool -change /Users/devqa/.pyenv/versions/3.9.7/lib/libpython3.9.dylib @rpath/libpython3.9.dylib python3.9
|
|
#
|
|
# The dynamic loader matches the install name id stored in the executable and the install name id of the library, so we want to
|
|
# change the libpython3.9.dylib install name id to something relative too:
|
|
#
|
|
# install_name_tool -id @rpath/libpython3.9.dylib Engine/Binaries/ThirdParty/Python3/Mac/lib/python3.9/libpython3.9.dylib
|
|
#
|
|
# The tokens '@rpath' and '@executable_path' are going to be replaced by the dynamic loader to locate files and then match
|
|
# the install name ids relative to the executable.
|
|
#
|
|
# The libpython3.9.dylib is loaded by UnrealEditor through a plugin. The UnrealEditor rpaths ensure we finds the plugin library
|
|
# first, then the plugin libraries refers to libpython3.11.dylib properly.
|
|
#
|
|
# The python core stuff is in libpython3.9.dylib, but several libraries are loaded 'on demand' when the intepreter encounter an import
|
|
# statement. Those dependencies are not visible by looking at libpython3.9.dylib. We need to be careful to ensure the module libraies
|
|
# are also correctly referred.
|
|
#
|
|
# --------------------------------------------------------------------------------
|
|
# Python tests suite (before running this script)
|
|
# --------------------------------------------------------------------------------
|
|
# Run Python unit tests (just after make install, before copying to UE tree). Those
|
|
# are the standard regression tests and they can be used to figure out if our build
|
|
# works outside the engine.
|
|
#
|
|
# WARNING: Few tests are sensible to the binary install names and from the root
|
|
# folder where the tests are invoked. From my understanding, this is just
|
|
# some limitations in the test itself.
|
|
#
|
|
# NOTE 1: When the full test suite ends, you get a summary. You will get the list of
|
|
# tests the passed, failed and the ones that were skipped. Review carefully
|
|
# the skipped tests list. Those were skipped because we didn't build the
|
|
# dependencies, because we didn't enable the resources (see below) or because
|
|
# they are platform specific. (I tried to run them all and check)
|
|
#
|
|
# NOTE 2: Some tests are disabled by default. If you scan the list of tests that
|
|
# were skipped, you can run some of them manually by enabling the 'resource'
|
|
# it needs with the -uall option. Bewared that some tests might be disabled
|
|
# for good reason, like the test_zipfile64 that use large amount of disk.
|
|
#
|
|
# Run the entire test suite from the binary install dir:
|
|
# cd $VolumeRoot/Deploy/Python3.11.8/bin
|
|
# ./python3.11 -m test
|
|
#
|
|
# To run a specific test:
|
|
# ./python3.11 -m test test_venv
|
|
#
|
|
# To run a specific test with verbose:
|
|
# ./python3.11 -m test -v test_venv
|
|
#
|
|
# To run a specific tests that was disabled becaue the resource wasn't enabled:
|
|
# See https://docs.python.org/3/library/test.html for more options.
|
|
# ./python3.11 -m test -uall -v test_socketserver
|
|
#
|
|
# --------------------------------------------------------------------------------
|
|
# Testing (after running this script)
|
|
# --------------------------------------------------------------------------------
|
|
#
|
|
# - Ensure the produced binaries are universal (Intel/Apple Sillicon ARM)
|
|
# - Run: lipo -archs Engine/Binaries/ThirdParty/Python3/Mac/lib/libpython3.11.dylib
|
|
# - Check for dependencies.
|
|
# - $:> cd Engine/Binaries/ThridParty/Python3/Mac/lib/python3.11/lib-dynload
|
|
# - $:> otool -L *.so'
|
|
# - Check the libraries dependencies for:
|
|
# - No dependencies are in /opt/usr/local -> Those are MacPort/Homebrew that user will likely not have.
|
|
# - Delete the $VolumeRoot/Deploy/Python3.11.8 -> This will ensure UE distribution doesn't have lib dependencies there.
|
|
# - Run the binary we plan to distribute as: Engine/Binaries/ThirdParty/Python3/Mac/bin/python3.11 --version
|
|
# - Run the unit tests distributed with python:
|
|
# - $:> cd Engine/Binaries/ThirdParty/Python3/Mac/lib/python3.11/test
|
|
# - $:> ../../../bin/python3.11 -m unittest discover
|
|
# - Launch UE5 EngineTest project
|
|
# - Go to menu Tools -> Test Automation
|
|
# - Search for python to discover all tests
|
|
# - Run all the tests (it is a good idea to run the test before the update to see if we have regression)
|
|
|
|
# --------------------------------------------------------------------------------
|
|
# Plugins
|
|
# --------------------------------------------------------------------------------
|
|
# - Some UE5 plugins likely needs to be rebuild when upgrading a minor version (3.7 to 3.9 for example)
|
|
# - USDImporter is one such plugin.
|
|
# - ML Deformer relies on PyTorch and may need some care.
|
|
#
|
|
# --------------------------------------------------------------------------------
|
|
# Tips:
|
|
# --------------------------------------------------------------------------------
|
|
# - To see all possible configure options: run ./configure --help
|
|
# - Read Mac/README.rst
|
|
# --------------------------------------------------------------------------------
|
|
|
|
python_src_dest_dir="`dirname \"$0\"`/Mac"
|
|
python_bin_dest_dir="`dirname \"$0\"`/../../../Binaries/ThirdParty/Python3/Mac"
|
|
python_bin_lib_dest_dir="$python_bin_dest_dir"/lib
|
|
python_src_dir="$VolumeRoot/Deploy/Python3.11.8"
|
|
python_exe_name=python3.11
|
|
python_lib_name=libpython3.11.dylib
|
|
|
|
|
|
if [ -d "$python_src_dir" ]
|
|
then
|
|
#
|
|
# Fixing up install names.
|
|
#
|
|
|
|
echo "Adding rpath to $python_exe_name."
|
|
install_name_tool -add_rpath @executable_path/../lib "$python_src_dir"/bin/$python_exe_name
|
|
|
|
echo "Fixing $python_exe_name dependencies: $python_src_dir/lib/$python_lib_name -> @rpath/$python_lib_name"
|
|
install_name_tool -change "$python_src_dir"/lib/$python_lib_name @rpath/$python_lib_name "$python_src_dir"/bin/$python_exe_name
|
|
|
|
echo "Fixing $python_lib_name install name id"
|
|
install_name_tool -id @rpath/$python_lib_name "$python_src_dir"/lib/$python_lib_name
|
|
|
|
#
|
|
# Deleting exiting destination folders from UE tree.
|
|
#
|
|
if [ -d "$python_src_dest_dir" ]
|
|
then
|
|
echo "Removing Existing Target Directory: $python_src_dest_dir"
|
|
rm -rf "$python_src_dest_dir"
|
|
fi
|
|
|
|
if [ -d "$python_bin_dest_dir" ]
|
|
then
|
|
echo "Removing Existing Target Directory: $python_bin_dest_dir"
|
|
rm -rf "$python_bin_dest_dir"
|
|
fi
|
|
|
|
#
|
|
# Copying local python intallation into UE tree.
|
|
#
|
|
echo "Copying Python: $python_src_dir"
|
|
|
|
mkdir -p "$python_src_dest_dir"/include
|
|
mkdir -p "$python_bin_dest_dir"/bin
|
|
mkdir -p "$python_bin_lib_dest_dir"
|
|
|
|
cp -R "$python_src_dir"/include/python3.11/* "$python_src_dest_dir"/include
|
|
cp -R "$python_src_dir"/bin/* "$python_bin_dest_dir"/bin
|
|
cp -R "$python_src_dir"/lib/* "$python_bin_lib_dest_dir"
|
|
|
|
cp -R "$python_src_dir"/lib/$python_lib_name "$python_bin_dest_dir"
|
|
chmod 755 "$python_bin_dest_dir"/$python_lib_name
|
|
|
|
#
|
|
# Copy TPS file back.
|
|
#
|
|
# if [ -f "$python_src_dest_dir"/../../../../Restricted/NoRedist/Source/ThirdParty/Python3/TPS/PythonMacBin.tps ]
|
|
# then
|
|
# cp -R "$python_src_dest_dir"/../../../../Restricted/NoRedist/Source/ThirdParty/Python3/TPS/PythonMacBin.tps "$python_bin_dest_dir"/
|
|
# fi
|
|
|
|
#
|
|
# Remove all symlinks (not a cross-platform thing).
|
|
#
|
|
echo "Processing Python symlinks: $python_dest_dir"
|
|
function remove_symlinks()
|
|
{
|
|
for file in $1/*
|
|
do
|
|
if [ -L "$file" ]
|
|
then
|
|
resolved_file="$1/`readlink \"$file\"`"
|
|
trimmed_file=".${file:$2}"
|
|
trimmed_resolved_file=".${resolved_file:$2}"
|
|
if [ -f "$resolved_file" ]
|
|
then
|
|
# for debugging
|
|
#echo " Removing symlink: $file -> $resolved_file"
|
|
echo " Removing symlink: $trimmed_file -> $trimmed_resolved_file"
|
|
rm -f "$file"
|
|
cp -R "$resolved_file" "$file"
|
|
else
|
|
echo "WARNING NOT FOUND: $resolved_file:"
|
|
fi
|
|
fi
|
|
|
|
if [ -d "$file" ]
|
|
then
|
|
remove_symlinks "$file" $2
|
|
fi
|
|
done
|
|
}
|
|
remove_symlinks "$python_bin_lib_dest_dir" ${#python_bin_lib_dest_dir}
|
|
|
|
function process_symlinks()
|
|
{
|
|
for file in $1/*
|
|
do
|
|
if [ -L "$file" ]
|
|
then
|
|
resolved_file="$1/`readlink \"$file\"`"
|
|
trimmed_file=".${file:$2}"
|
|
trimmed_resolved_file=".${resolved_file:$2}"
|
|
if [ -f "$resolved_file" ]
|
|
then
|
|
# for debugging
|
|
#echo " Processing symlink: $file -> $resolved_file"
|
|
echo " Processing symlink: $trimmed_file -> $trimmed_resolved_file"
|
|
rm -f "$file"
|
|
cp "$resolved_file" "$file"
|
|
else
|
|
echo "WARNING NOT FOUND: $resolved_file:"
|
|
fi
|
|
fi
|
|
|
|
if [ -d "$file" ]
|
|
then
|
|
process_symlinks "$file" $2
|
|
fi
|
|
done
|
|
}
|
|
process_symlinks "$python_bin_dest_dir" ${#python_bin_dest_dir}
|
|
|
|
#
|
|
# Remove any temporary files that were moved
|
|
#
|
|
function remove_obj_files()
|
|
{
|
|
for file in $1/*
|
|
do
|
|
if [ "${file}" != "${file%.pyc}" ] || [ "${file}" != "${file%.pyo}" ]
|
|
then
|
|
trimmed_file=".${file:$2}"
|
|
#echo " Removing: $trimmed_file"
|
|
rm -f "$file"
|
|
fi
|
|
|
|
if [ -d "$file" ]
|
|
then
|
|
remove_obj_files "$file" $2
|
|
fi
|
|
done
|
|
}
|
|
remove_obj_files "$python_bin_lib_dest_dir" ${#python_bin_lib_dest_dir}
|
|
|
|
function copy_openssl_libs()
|
|
{
|
|
# this was needed when using latest python3.9, their latest hashlib
|
|
# uses some of libssl and libcrypto functions (instead of their own anymore)
|
|
# TODO: see if this can be statically linked next time
|
|
|
|
# might need a peek at lib*.x.y.z.dylib for the actual hard coded path
|
|
openssl_lib_dir=/usr/local/opt/openssl/lib
|
|
|
|
# saving these instructions here on how to do this for future reference
|
|
cp "$openssl_lib_dir"/libssl.1.0.0.dylib "$python_bin_dest_dir"
|
|
cp "$openssl_lib_dir"/libcrypto.1.0.0.dylib "$python_bin_dest_dir"
|
|
|
|
install_name_tool -id "@rpath/libssl.1.0.0.dylib" "$python_bin_dest_dir"/libssl.1.0.0.dylib
|
|
install_name_tool -change "$openssl_lib_dir/libcrypto.1.0.0.dylib" "@executable_path/../libcrypto.1.0.0.dylib" "$python_bin_dest_dir"/libssl.1.0.0.dylib
|
|
|
|
install_name_tool -id "@rpath/libcrypto.1.0.0.dylib" "$python_bin_dest_dir"/libcrypto.1.0.0.dylib
|
|
|
|
# finally:
|
|
install_name_tool -change "$openssl_lib_dir/libssl.1.0.0.dylib" "@executable_path/../libssl.1.0.0.dylib" "$python_bin_dest_dir"/lib/python3.11/lib-dynload/_hashlib.so
|
|
install_name_tool -change "$openssl_lib_dir/libcrypto.1.0.0.dylib" "@executable_path/../libcrypto.1.0.0.dylib" "$python_bin_dest_dir"/lib/python3.11/lib-dynload/_hashlib.so
|
|
}
|
|
# disabling this - again, for future reference
|
|
#copy_openssl_libs
|
|
|
|
else
|
|
echo "Python Source Directory Missing: $python_src_dir"
|
|
fi
|
|
|
|
if [ ! -f "$python_src_dest_dir"/../../../../Restricted/NoRedist/Source/ThirdParty/Python/TPS/PythonMacBin.tps ]
|
|
then
|
|
echo "."
|
|
echo "WARNING: restore (i.e. revert) deleted $python_bin_dest_dir/PythonMacBin.tps before checking in"
|
|
echo "."
|
|
fi
|