From 196767124650c05c528c8d2418795965c5f68aae Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Fri, 12 Nov 2021 09:55:43 -0600 Subject: [PATCH 01/23] Set 10.11 deployment target --- build-ffmpeg-descript.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index ebbb9d4b..2ad34988 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -45,6 +45,7 @@ def buildFFmpeg(script_dir, log_file): env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc env['VERBOSE'] = 'yes' + env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') From 5af67732eed781b88c37819c14d2e19b7c1b8813 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 12 Nov 2021 11:11:23 -0600 Subject: [PATCH 02/23] conditionalize 10.11 target to x86_64 --- build-ffmpeg-descript.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 2ad34988..05fdeb81 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -45,7 +45,8 @@ def buildFFmpeg(script_dir, log_file): env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc env['VERBOSE'] = 'yes' - env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' + if platform.machine() == 'x86_64': + env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') From 75562a1e83afd4d9e36be54de8256f9c47e67b61 Mon Sep 17 00:00:00 2001 From: Markus Perl <1220081+markus-perl@users.noreply.github.com> Date: Sat, 13 Nov 2021 10:28:48 +0100 Subject: [PATCH 03/23] PCRE Mirror fix --- build-ffmpeg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index af8875d0..e8001b5f 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.31 +SCRIPT_VERSION=1.32 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" @@ -622,13 +622,13 @@ if command_exists "python3"; then execute ./waf install build_done "serd" "0.30.10" fi - if build "pcre" "8.44"; then - download "https://ftp.pcre.org/pub/pcre/pcre-8.44.tar.gz" "pcre-8.44.tar.gz" + if build "pcre" "8.45"; then + download "https://altushost-swe.dl.sourceforge.net/project/pcre/pcre/8.45/pcre-8.45.tar.gz" "pcre-8.45.tar.gz" execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static execute make -j $MJOBS execute make install - build_done "pcre" "8.44" + build_done "pcre" "8.45" fi if build "sord" "0.16.8"; then download "https://gitlab.com/drobilla/sord/-/archive/v0.16.8/sord-v0.16.8.tar.gz" "sord-v0.16.8.tar.gz" From 7ae21ff585f5264934710f1c6deeb19707b4bab0 Mon Sep 17 00:00:00 2001 From: Markus Perl <1220081+markus-perl@users.noreply.github.com> Date: Sat, 13 Nov 2021 13:39:38 +0100 Subject: [PATCH 04/23] webp new version 1.2.1 --- build-ffmpeg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index e8001b5f..4ccd46cb 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -758,8 +758,8 @@ fi # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS CPPFLAGS= -if build "libwebp" "1.2.0"; then - download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.0.tar.gz" "libwebp-1.2.0.tar.gz" +if build "libwebp" "1.2.1"; then + download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static make_dir build cd build || exit @@ -767,7 +767,7 @@ if build "libwebp" "1.2.0"; then execute make -j $MJOBS execute make install - build_done "libwebp" "1.2.0" + build_done "libwebp" "1.2.1" fi CONFIGURE_OPTIONS+=("--enable-libwebp") From 5210332bda02988e05779f6afad8337750d49b56 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Sun, 14 Nov 2021 22:11:47 -0600 Subject: [PATCH 05/23] Fix deployment target --- build-ffmpeg | 4 ++-- build-ffmpeg-descript.py | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 33548668..ced284fb 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -869,7 +869,7 @@ if build "libtheora" "1.1.1"; then chmod +x configure.patched mv configure.patched configure if $SHARED_LIBRARIES; then - if $MACOSX_M1; then + if $MACOS_M1; then # fix Libtool bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63610 export MACOSX_DEPLOYMENT_TARGET=10.11 fi @@ -881,7 +881,7 @@ if build "libtheora" "1.1.1"; then execute make install if $SHARED_LIBRARIES; then - if $MACOSX_M1; then + if $MACOS_M1; then # restore from Line 26 export MACOSX_DEPLOYMENT_TARGET=11.0 fi diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 05fdeb81..42d20616 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -29,6 +29,7 @@ workspace_dir = os.path.join(cwd, 'workspace') workspace_bin_dir = os.path.join(workspace_dir, 'bin') workspace_lib_dir = os.path.join(workspace_dir, 'lib') +deployment_target = '11.0' if platform.machine() == 'arm64' else '10.11' # # Keep track of which libraries are copied, skipped, or missing @@ -45,8 +46,7 @@ def buildFFmpeg(script_dir, log_file): env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc env['VERBOSE'] = 'yes' - if platform.machine() == 'x86_64': - env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' + env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') @@ -101,11 +101,31 @@ def copyOrGenerateSymbolFiles(source, dest, log_file): for fileref in pathlib.Path(source + '/').glob('**/*.so*'): copyOrGenerateSymbolFile(str(fileref), dest, log_file) +def readDeploymentTarget(src_file) -> str: + args = ['/usr/bin/otool', '-l', src_file] + otool_proc = subprocess.Popen(args, stdout=subprocess.PIPE) + inLoaderCommand = False + for line in otool_proc.stdout: + ln = line.decode('utf-8').strip() + if inLoaderCommand: + if ln.startswith('minos') or ln.startswith('version'): + return ln.split(' ')[1] + if ln.startswith('sdk'): + continue + elif 'LC_VERSION_MIN_MACOSX' in ln or 'LC_BUILD_VERSION' in ln: + inLoaderCommand = True + + return '' + + # # Copies a library and its corresponding .dSYM bundle # (if present) # def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): + this_deployment_target = readDeploymentTarget(src_file) + assert this_deployment_target == deployment_target, '{0} wrong deployment target {1}'.format(src_file, this_deployment_target) + dest_file = os.path.join(dest_folder, os.path.basename(src_file)) # copy file From d3e9e48b06c5821031b26c5033253bcd80638559 Mon Sep 17 00:00:00 2001 From: Markus Perl <1220081+markus-perl@users.noreply.github.com> Date: Mon, 15 Nov 2021 08:40:57 +0100 Subject: [PATCH 06/23] Try to fix MacOS build --- build-ffmpeg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 4ccd46cb..def0d0d6 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -755,12 +755,12 @@ if build "libpng" "1.6.37"; then execute make install build_done "libpng" "1.6.37" fi -# libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS -CPPFLAGS= if build "libwebp" "1.2.1"; then + # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS + CPPFLAGS= download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib make_dir build cd build || exit execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ From 349870c9545b9ca4434735b0fcf1af0560d49c35 Mon Sep 17 00:00:00 2001 From: Markus Perl <1220081+markus-perl@users.noreply.github.com> Date: Mon, 15 Nov 2021 12:53:33 +0100 Subject: [PATCH 07/23] MacOs fix --- build-ffmpeg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index def0d0d6..7117dfc4 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.32 +SCRIPT_VERSION=1.33 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" @@ -336,7 +336,6 @@ if build "zlib" "1.2.11"; then execute make install build_done "zlib" "1.2.11" fi -LDFLAGS+=" -L/zlib/lib" if build "m4" "1.4.19"; then download "https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.gz" From 1ac69a4d8087267101d614c0dd20fe2de474a3f1 Mon Sep 17 00:00:00 2001 From: Markus Perl <1220081+markus-perl@users.noreply.github.com> Date: Mon, 15 Nov 2021 14:15:14 +0100 Subject: [PATCH 08/23] MacOs fix --- build-ffmpeg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index 7117dfc4..d1317fb9 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -759,7 +759,7 @@ if build "libwebp" "1.2.1"; then # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS CPPFLAGS= download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --disable-gif --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib make_dir build cd build || exit execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ From a78f120b07db63d2a0330e759c858b88ec6c7318 Mon Sep 17 00:00:00 2001 From: Markus Perl <1220081+markus-perl@users.noreply.github.com> Date: Tue, 30 Nov 2021 09:59:16 +0100 Subject: [PATCH 09/23] Mac fix --- build-ffmpeg | 75 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index d1317fb9..1839e4c0 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -93,6 +93,10 @@ download() { make_dir "$DOWNLOAD_PATH/$TARGETDIR" + if [[ "$DOWNLOAD_FILE" == *"patch"* ]]; then + return + fi + if [ -n "$3" ]; then if ! tar -xvf "$DOWNLOAD_PATH/$DOWNLOAD_FILE" -C "$DOWNLOAD_PATH/$TARGETDIR" 2>/dev/null >/dev/null; then echo "Failed to extract $DOWNLOAD_FILE" @@ -305,6 +309,17 @@ fi ## build tools ## +if build "giflib" "5.2.1"; then + download "https://sourceforge.net/projects/giflib/files/giflib-5.2.1.tar.gz" + if [[ "$OSTYPE" == "darwin"* ]]; then + download "https://sourceforge.net/p/giflib/bugs/_discuss/thread/4e811ad29b/c323/attachment/Makefile.patch" + execute patch "${PACKAGES}/giflib-5.2.1/Makefile" ${PACKAGES}/Makefile.patch"" + fi + execute make -j $MJOBS + execute make PREFIX="${WORKSPACE}" install + build_done "giflib" "5.2.1" +fi + if build "pkg-config" "0.29.2"; then download "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" execute ./configure --silent --prefix="${WORKSPACE}" --with-pc-path="${WORKSPACE}"/lib/pkgconfig --with-internal-glib @@ -385,12 +400,12 @@ if $NONFREE_AND_GPL; then CONFIGURE_OPTIONS+=("--enable-openssl") fi -if build "cmake" "3.21.2"; then - download "https://cmake.org/files/LatestRelease/cmake-3.21.2.tar.gz" +if build "cmake" "3.22.0"; then + download "https://github.com/Kitware/CMake/releases/download/v3.22.0/cmake-3.22.0.tar.gz" execute ./configure --prefix="${WORKSPACE}" --parallel="${MJOBS}" -- -DCMAKE_USE_OPENSSL=OFF execute make -j $MJOBS execute make install - build_done "cmake" "3.21.2" + build_done "cmake" "3.22.0" fi ## @@ -408,15 +423,17 @@ if command_exists "python3"; then fi done fi - if build "dav1d" "0.9.2"; then - download "https://code.videolan.org/videolan/dav1d/-/archive/0.9.2/dav1d-0.9.2.tar.gz" - make_dir build - execute meson build --prefix="${WORKSPACE}" --buildtype=release --default-library=static --libdir="${WORKSPACE}"/lib - execute ninja -C build - execute ninja -C build install - build_done "dav1d" "0.9.2" + if command_exists "meson"; then + if build "dav1d" "0.9.2"; then + download "https://code.videolan.org/videolan/dav1d/-/archive/0.9.2/dav1d-0.9.2.tar.gz" + make_dir build + execute meson build --prefix="${WORKSPACE}" --buildtype=release --default-library=static --libdir="${WORKSPACE}"/lib + execute ninja -C build + execute ninja -C build install + build_done "dav1d" "0.9.2" + fi + CONFIGURE_OPTIONS+=("--enable-libdav1d") fi - CONFIGURE_OPTIONS+=("--enable-libdav1d") fi if ! $MACOS_M1; then @@ -738,12 +755,12 @@ fi ## image library ## -if build "libtiff" "4.3.0"; then - download "https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static +if build "libtiff" "4.2.0"; then + download "https://download.osgeo.org/libtiff/tiff-4.2.0.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --disable-dependency-tracking --disable-lzma --disable-webp --disable-zstd --without-x execute make -j $MJOBS execute make install - build_done "libtiff" "4.3.0" + build_done "libtiff" "4.2.0" fi if build "libpng" "1.6.37"; then download "https://sourceforge.net/projects/libpng/files/libpng16/1.6.37/libpng-1.6.37.tar.gz/download?use_mirror=gigenet" "libpng-1.6.37.tar.gz" @@ -755,21 +772,23 @@ if build "libpng" "1.6.37"; then build_done "libpng" "1.6.37" fi -if build "libwebp" "1.2.1"; then - # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS - CPPFLAGS= - download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --disable-gif --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib - make_dir build - cd build || exit - execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ - execute make -j $MJOBS - execute make install +## does not compile on monterey -> _PrintGifError +if [[ "$OSTYPE" != "darwin"* ]]; then + if build "libwebp" "1.2.1"; then + # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS + CPPFLAGS= + download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --disable-dependency-tracking --disable-gl --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib + make_dir build + cd build || exit + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ + execute make -j $MJOBS + execute make install - build_done "libwebp" "1.2.1" + build_done "libwebp" "1.2.1" + fi + CONFIGURE_OPTIONS+=("--enable-libwebp") fi -CONFIGURE_OPTIONS+=("--enable-libwebp") - ## ## other library ## From a0be4b6d9afcac87093fadaebd370563d3045965 Mon Sep 17 00:00:00 2001 From: markus-perl <1220081+markus-perl@users.noreply.github.com> Date: Thu, 2 Dec 2021 08:07:09 +0100 Subject: [PATCH 10/23] Version update --- build-ffmpeg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index 1839e4c0..4601ab6e 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.33 +SCRIPT_VERSION=1.34 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" From 6728e84e078b7cd5228d49ec1f6de4770be6f3a6 Mon Sep 17 00:00:00 2001 From: markus-perl <1220081+markus-perl@users.noreply.github.com> Date: Thu, 2 Dec 2021 08:07:38 +0100 Subject: [PATCH 11/23] Version update --- build-ffmpeg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index 4601ab6e..1839e4c0 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.34 +SCRIPT_VERSION=1.33 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" From e7a117a2ca31999b3cbe65d96dd388ef55060214 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Thu, 2 Dec 2021 23:05:58 -0600 Subject: [PATCH 12/23] Move `build-ffmpeg-descript.py` to a `descript` subfolder --- descript/.gitignore | 1 + .../build-ffmpeg-descript.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 descript/.gitignore rename build-ffmpeg-descript.py => descript/build-ffmpeg-descript.py (97%) diff --git a/descript/.gitignore b/descript/.gitignore new file mode 100644 index 00000000..28757cf4 --- /dev/null +++ b/descript/.gitignore @@ -0,0 +1 @@ +mac \ No newline at end of file diff --git a/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py similarity index 97% rename from build-ffmpeg-descript.py rename to descript/build-ffmpeg-descript.py index 42d20616..f70ca163 100755 --- a/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -25,8 +25,9 @@ # Constants # cwd = os.path.dirname(os.path.realpath(__file__)) -packages_dir = os.path.join(cwd, 'packages') -workspace_dir = os.path.join(cwd, 'workspace') +base_dir = pathlib.Path(cwd).parent.absolute() +packages_dir = os.path.join(base_dir, 'packages') +workspace_dir = os.path.join(base_dir, 'workspace') workspace_bin_dir = os.path.join(workspace_dir, 'bin') workspace_lib_dir = os.path.join(workspace_dir, 'lib') deployment_target = '11.0' if platform.machine() == 'arm64' else '10.11' @@ -236,7 +237,7 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): # def readVersion() -> str: result = '' - with open(os.path.join(cwd, 'build-ffmpeg')) as f: + with open(os.path.join(base_dir, 'build-ffmpeg')) as f: lines = f.readlines() for line in lines: if line.startswith('SCRIPT_VERSION='): @@ -253,7 +254,7 @@ def getPlatformMachineVersion() -> str: # # def main(): - output_dir = os.path.join(workspace_dir, 'mac', platform.machine()) + output_dir = os.path.join(cwd, 'mac', platform.machine()) if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) @@ -267,7 +268,7 @@ def main(): build_ffmpeg_log_file.write('=======================\n') # Run the script - buildFFmpeg(cwd, build_ffmpeg_log_file) + buildFFmpeg(base_dir, build_ffmpeg_log_file) # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') From 1d395f8957bfc97927df35964b42ba5d5dc688ec Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Thu, 2 Dec 2021 23:45:44 -0600 Subject: [PATCH 13/23] Improve bundling to match new features from Descript electron build script --- descript/build-ffmpeg-descript.py | 69 +++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index f70ca163..96bc634c 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -19,7 +19,7 @@ import shutil import subprocess import sys -from zipfile import ZipFile +import zipfile # # Constants @@ -250,25 +250,58 @@ def readVersion() -> str: def getPlatformMachineVersion() -> str: return sys.platform + '-' + platform.machine() + '.' + readVersion() + +# +# +# +def generateChecksum(output_folder): + """ + Calculates checksums for every file in `output_folder` + """ + + checksums = set() + + # calculate checksums for all files + for (dirpath, dirnames, filenames) in os.walk(output_folder): + for file in filenames: + args = ['shasum', '-a', '256', os.path.join(dirpath, file)] + output = subprocess.check_output(args) + checksum = output.decode('utf-8').strip() + + # replace absolute path to just filename + # From: '0a88d3f97f356c6a42449fd548f9b586f565899144849019014e36c7683b745e /Users/cvanwink/Source/git/electron/src/out/Testing/dist.zip' + # To: '0a88d3f97f356c6a42449fd548f9b586f565899144849019014e36c7683b745e *electron-v13.1.6-darwin-x64.zip' + checksum = checksum.replace(os.path.join(dirpath, ''), '*') + checksums.add(checksum) + break + + # Write Checksums to file + checksum_file_path = os.path.join(output_folder, 'SHAMSUM256.txt') + checksum_file = open(checksum_file_path, 'w') + for checksum in checksums: + checksum_file.write(f'{checksum}\n') + checksum_file.close() + # # # def main(): - output_dir = os.path.join(cwd, 'mac', platform.machine()) + output_dir = os.path.join(cwd, 'mac') if os.path.exists(output_dir): shutil.rmtree(output_dir) - os.makedirs(output_dir) + temp_dir = os.path.join(output_dir, platform.machine()) + os.makedirs(temp_dir) # create a log file for the build-ffmpeg command for build archival purposes log_file_name = 'build-ffmpeg-' + getPlatformMachineVersion() + '.log.txt' - build_ffmpeg_log_file_path = os.path.join(os.path.dirname(output_dir), log_file_name) - build_ffmpeg_log_file = open(build_ffmpeg_log_file_path, 'w') + log_file_path = os.path.join(output_dir, log_file_name) + build_ffmpeg_log_file = open(log_file_path, 'w') build_ffmpeg_log_file.write('Begin build-ffmpeg-descript.py\n') build_ffmpeg_log_file.write('=======================\n') # Run the script - buildFFmpeg(base_dir, build_ffmpeg_log_file) + #buildFFmpeg(base_dir, build_ffmpeg_log_file) # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') @@ -283,12 +316,12 @@ def main(): build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) copyOrGenerateSymbolFile(executable_path, workspace_bin_dir, build_ffmpeg_log_file) - copyLibraryAndDependencies(executable_path, output_dir, build_ffmpeg_log_file) + copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file) # check that the copied file is runnable build_ffmpeg_log_file.write('\nChecking ' + executable + '\n') build_ffmpeg_log_file.write('=======================\n') - args = [os.path.join(output_dir, executable), '-version'] + args = [os.path.join(temp_dir, executable), '-version'] build_ffmpeg_log_file.write(' '.join(args) + '\n') output = subprocess.check_output(args) build_ffmpeg_log_file.write(output.decode('utf-8')) @@ -296,7 +329,7 @@ def main(): # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), - os.path.join(output_dir, 'include')) + os.path.join(temp_dir, 'include')) build_ffmpeg_log_file.write('\nLibrary Info\n') build_ffmpeg_log_file.write('=======================\n') @@ -315,8 +348,9 @@ def main(): # bundle up the third-party source # grab each .tar.* from the packages folder - packages_zip_name = '-'.join(executables) + '-packages-' + getPlatformMachineVersion() + '.zip' - with ZipFile(os.path.join(os.path.dirname(output_dir), packages_zip_name), 'w') as myzip: + shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' + packages_zip_name = f'{pathlib.Path(shared_zip_name).stem}-packages.zip' + with zipfile.ZipFile(os.path.join(output_dir, packages_zip_name), 'w', zipfile.ZIP_DEFLATED) as myzip: archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') for archive in sorted(archives, key=lambda s: str(s).lower()): build_ffmpeg_log_file.write(os.path.join('packages', archive.name) + '\n') @@ -326,16 +360,25 @@ def main(): build_ffmpeg_log_file.write('=======================\n') # bundle up the build artifacts - os.chdir(output_dir) - shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' + os.chdir(temp_dir) + dest_file = os.path.join(output_dir, shared_zip_name) args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] build_ffmpeg_log_file.write(' '.join(args) + '\n') subprocess.check_output(args) + + shutil.rmtree(temp_dir) build_ffmpeg_log_file.write('\nEnd of build-ffmpeg-descript.py\n') build_ffmpeg_log_file.write('=======================\n') build_ffmpeg_log_file.close() + # zip up log file + with zipfile.ZipFile(os.path.splitext(log_file_path)[0] + '.zip', 'w', zipfile.ZIP_DEFLATED) as myzip: + myzip.write(log_file_path, os.path.basename(log_file_path)) + os.remove(log_file_path) + + generateChecksum(output_dir) + # # entry # From 13ee124567252a9b98f21207e82346ce8440e20c Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Thu, 2 Dec 2021 23:48:24 -0600 Subject: [PATCH 14/23] Remove change which was mistakenly staged and committed --- descript/build-ffmpeg-descript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 96bc634c..669b9858 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -6,7 +6,7 @@ (1) Call build-ffmpeg with the build command (2) Copy or generate dSYM symbol files to the workspace folder -(3) Copy executables from the workspace folder and all built dependencies to platform outputfolder +(3) Copy executables from the workspace folder and all built dependencies to platform output folder (4) Fix dyld ids and loader paths for all built libraries (5) Zip up the build artifacts ''' @@ -301,7 +301,7 @@ def main(): build_ffmpeg_log_file.write('=======================\n') # Run the script - #buildFFmpeg(base_dir, build_ffmpeg_log_file) + buildFFmpeg(base_dir, build_ffmpeg_log_file) # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') From f2bc24f77b4b8f3b0cdfde5019daf8a5d388dbae Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 10:31:18 -0600 Subject: [PATCH 15/23] Separate symbols into their own zip file --- descript/build-ffmpeg-descript.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 669b9858..f59df371 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -291,9 +291,13 @@ def main(): shutil.rmtree(output_dir) temp_dir = os.path.join(output_dir, platform.machine()) os.makedirs(temp_dir) + symbol_temp_dir = os.path.join(output_dir, platform.machine() + '-symbols') + + executables = ['ffmpeg', 'ffprobe'] + base_artifact_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() # create a log file for the build-ffmpeg command for build archival purposes - log_file_name = 'build-ffmpeg-' + getPlatformMachineVersion() + '.log.txt' + log_file_name = base_artifact_name + '-log.txt' log_file_path = os.path.join(output_dir, log_file_name) build_ffmpeg_log_file = open(log_file_path, 'w') @@ -306,11 +310,13 @@ def main(): # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') build_ffmpeg_log_file.write('=======================\n') - copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir, build_ffmpeg_log_file) + copyOrGenerateSymbolFiles(packages_dir, symbol_temp_dir, build_ffmpeg_log_file) + symbol_file_name = base_artifact_name + '-symbols' + shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) + shutil.rmtree(symbol_temp_dir) # Generate dSYM files for each executable # and copy their dependencies - executables = ['ffmpeg', 'ffprobe'] for executable in executables: build_ffmpeg_log_file.write('\nCopying & Linking ' + executable + '\n') build_ffmpeg_log_file.write('=======================\n') @@ -348,8 +354,7 @@ def main(): # bundle up the third-party source # grab each .tar.* from the packages folder - shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' - packages_zip_name = f'{pathlib.Path(shared_zip_name).stem}-packages.zip' + packages_zip_name = base_artifact_name + '-packages.zip' with zipfile.ZipFile(os.path.join(output_dir, packages_zip_name), 'w', zipfile.ZIP_DEFLATED) as myzip: archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') for archive in sorted(archives, key=lambda s: str(s).lower()): @@ -361,6 +366,7 @@ def main(): # bundle up the build artifacts os.chdir(temp_dir) + shared_zip_name = base_artifact_name + '.zip' dest_file = os.path.join(output_dir, shared_zip_name) args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] build_ffmpeg_log_file.write(' '.join(args) + '\n') From 606d18db1d65f1fd5bd05044b93c4e6e49b17baa Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 10:41:20 -0600 Subject: [PATCH 16/23] Add basic README file for Descript folder --- descript/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 descript/README.md diff --git a/descript/README.md b/descript/README.md new file mode 100644 index 00000000..70c884e1 --- /dev/null +++ b/descript/README.md @@ -0,0 +1,30 @@ +# Build FFmpeg for Descript + +## Author / Contact: + - [Charles Van Winkle](https://github.com/cvanwinkle) + - [Steve Rubin](https://github.com/srubin) + +## Instructions +- Run `build-ffmpeg-descript.py` + + +## Build Overview +The build script automates the following basic operations. +- Creates a log file to archive the compiler/linker and packaging steps +- Runs the modified `buildFFmpeg` shell script, outputs to log file +- Recursively generates or copies `.dSYM` symbol files for each dependency into a `.zip` file + - Also fixes `dlyd` loader paths for each dependency +- Checks that each executable (i.e. `ffmpeg` & `ffprobe`) are runnable +- Copies `includes` header folder +- Checks for any linked dependencies which are linked to locations on the build machine and not present in the archive bundle +- Archives the upstream tar bundles for each `ffmpeg` component + - This is important in case upstream FTP or source servers go offline in the future +- Generates checksum for each created artifact (build, symbols, packages, log) + +## Development +Known issues: +- . + +## Patches + + From 58238849b64d90807946eee4db96700d435c446e Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 10:54:05 -0600 Subject: [PATCH 17/23] Switch comments style to match docstring format --- descript/build-ffmpeg-descript.py | 73 ++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index f59df371..48daab1e 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -40,9 +40,12 @@ missing_libs = set() # -# builds FFmpeg and logs output to build-ffmpeg.log.txt +# # def buildFFmpeg(script_dir, log_file): + """ + builds FFmpeg and logs output to `log_file` + """ # set environment variables env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc @@ -60,11 +63,14 @@ def buildFFmpeg(script_dir, log_file): subprocess.call(args, env=env, stdout=log_file) # -# Copies symbol file to the workspace destination -# skips symlinks to avoid duplication -# Copies entire dSYM packages for dylib files already within .dSYM packages +# # def copyOrGenerateSymbolFile(file, dest, log_file): + """ + Copies a single symbol file to the workspace destination + skips symlinks to avoid duplication + Copies entire `dSYM` packages for `dylib` files already within `.dSYM` packages + """ fileref = pathlib.Path(file) if not fileref.is_symlink(): symbolFileName = fileref.name + '.dSYM' @@ -92,17 +98,27 @@ def copyOrGenerateSymbolFile(file, dest, log_file): subprocess.call(args, stdout=log_file) # -# Copies symbol files to the workspace destination -# skips symlinks to avoid duplication -# Copies entire dSYM packages for dylib files already within .dSYM packages +# # def copyOrGenerateSymbolFiles(source, dest, log_file): + """ + Recursively copies symbol files to the workspace destination + skips symlinks to avoid duplication + Copies entire `dSYM` packages for `dylib` files already within `.dSYM` packages + """ for fileref in pathlib.Path(source + '/').glob('**/*.dylib'): copyOrGenerateSymbolFile(str(fileref), dest, log_file) for fileref in pathlib.Path(source + '/').glob('**/*.so*'): copyOrGenerateSymbolFile(str(fileref), dest, log_file) +# +# +# def readDeploymentTarget(src_file) -> str: + """ + Reads the deployment target of a binary + :return: something like `'10.11'` or an empty string + """ args = ['/usr/bin/otool', '-l', src_file] otool_proc = subprocess.Popen(args, stdout=subprocess.PIPE) inLoaderCommand = False @@ -120,10 +136,13 @@ def readDeploymentTarget(src_file) -> str: # -# Copies a library and its corresponding .dSYM bundle -# (if present) +# # def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): + """ + Copies a library and its corresponding `.dSYM` bundle + (if present) + """ this_deployment_target = readDeploymentTarget(src_file) assert this_deployment_target == deployment_target, '{0} wrong deployment target {1}'.format(src_file, this_deployment_target) @@ -144,21 +163,26 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): shutil.copytree(src_symbol_package, dest_symbol_package) # -# Helper function to get a base name of a library -# without version numbers +# # def getFilenameWithoutVersion(file_name) -> str: - result = file_name.split('.')[0] - # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if 'libSDL2' in result: - result = 'libSDL2' - return result + """ + :return: `'libSDL2'` for something like `'libSDL2-2.0.0.dylib'` + """ + result = file_name.split('.')[0] + # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) + if 'libSDL2' in result: + result = 'libSDL2' + return result # -# Recursive function to copy a library and its (non-system) dependencies -# also fixes loader paths for each library +# # def copyLibraryAndDependencies(src_file, dest_folder, log_file): + """ + Recursive function to copy a library and its (non-system) dependencies + also fixes loader paths for each library to be `@loader_path` + """ dest_file = os.path.join(dest_folder, os.path.basename(src_file)) @@ -233,9 +257,13 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): subprocess.check_output(args) # -# Read the version string from ./build-ffmpeg +# # def readVersion() -> str: + """ + Reads the version string from ../build-ffmpeg + :return: something like `'1.31rc1'` + """ result = '' with open(os.path.join(base_dir, 'build-ffmpeg')) as f: lines = f.readlines() @@ -245,9 +273,12 @@ def readVersion() -> str: return result # -# Returns a string like darwin-x86_64.1.31rc2 +# # def getPlatformMachineVersion() -> str: + """ + :return: a string like `'darwin-x86_64.1.31rc2'` + """ return sys.platform + '-' + platform.machine() + '.' + readVersion() @@ -257,8 +288,8 @@ def getPlatformMachineVersion() -> str: def generateChecksum(output_folder): """ Calculates checksums for every file in `output_folder` + and puts it in a `SHAMSUM256.txt` file """ - checksums = set() # calculate checksums for all files From ccfaa1652ce21e0d65b6fed4d0e0302c6338279e Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 20:34:58 -0600 Subject: [PATCH 18/23] More copy fixes --- descript/build-ffmpeg-descript.py | 60 ++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 48daab1e..80bd0118 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -165,15 +165,42 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): # # # -def getFilenameWithoutVersion(file_name) -> str: +def getFileBaseNameWithoutVersion(file_path) -> str: """ - :return: `'libSDL2'` for something like `'libSDL2-2.0.0.dylib'` + :return: `'libpostproc'` for something like `'/foo/bar/libpostproc.55.9.100.dylib'` """ - result = file_name.split('.')[0] + base_name = os.path.basename(file_path) + base_name = base_name.split('.')[0] # keep everything before first '.' # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if 'libSDL2' in result: - result = 'libSDL2' - return result + if base_name.startswith('libSDL2'): + base_name = 'libSDL2' + return base_name + +# +# +# +def getVersionVariantsForFile(file_path) -> list[str]: + """ + Returns the following three files for any one of the file paths provided: + `'.../ffmpeg-build-script/workspace/lib/libavcodec.58.134.100.dylib'` + `'.../ffmpeg-build-script/workspace/lib/libavcodec.58.dylib'` + `'.../ffmpeg-build-script/workspace/lib/libavcodec.dylib'` + + """ + result = set() + result.add(file_path) + if (pathlib.Path(file_path).is_symlink()): + result.add(os.path.realpath(file_path)) + + dependency_name_without_version = getFileBaseNameWithoutVersion(file_path) + unversioned_dependency_base_name = os.path.join( + os.path.dirname(file_path), + dependency_name_without_version) + + for variant in glob.glob(unversioned_dependency_base_name + r'.*dylib'): + result.add(variant) + + return list(result) # # @@ -206,8 +233,9 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): src_dependency_file = match[0] # fix incorrect usage of @rpath - if src_dependency_file.startswith('@rpath/'): - fixed_path = os.path.join(workspace_lib_dir, src_dependency_file[7:]) + rpath_token = '@rpath/' + if src_dependency_file.startswith(rpath_token): + fixed_path = os.path.join(workspace_lib_dir, src_dependency_file[len(rpath_token):]) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': fixed_path}) src_dependency_file = fixed_path @@ -224,17 +252,14 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): if not src_dependency_file in copied_libs: if src_dependency_file != dest_dependency_path: # Copy each version variant file (often symlinks) - dependency_name_without_version = getFilenameWithoutVersion(src_dependency_file) - unversioned_dependency_base_name = os.path.join(os.path.dirname(src_dependency_file), dependency_name_without_version) - for variant_src_file in glob.glob(unversioned_dependency_base_name + r'*.dylib'): + for variant_src_file in getVersionVariantsForFile(src_dependency_file): copyLibraryAndSymbolPackage(variant_src_file, dest_folder, False) variant_dest_file = os.path.join(dest_folder, os.path.basename(variant_src_file)) copied_libs.add(variant_src_file) copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies - if (os.path.exists(unversioned_dependency_base_name + '.dylib')): - copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder, log_file) + copyLibraryAndDependencies(os.path.relpath(src_dependency_file), dest_folder, log_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: @@ -342,9 +367,6 @@ def main(): build_ffmpeg_log_file.write('\nGenerating Symbols\n') build_ffmpeg_log_file.write('=======================\n') copyOrGenerateSymbolFiles(packages_dir, symbol_temp_dir, build_ffmpeg_log_file) - symbol_file_name = base_artifact_name + '-symbols' - shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) - shutil.rmtree(symbol_temp_dir) # Generate dSYM files for each executable # and copy their dependencies @@ -352,7 +374,7 @@ def main(): build_ffmpeg_log_file.write('\nCopying & Linking ' + executable + '\n') build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) - copyOrGenerateSymbolFile(executable_path, workspace_bin_dir, build_ffmpeg_log_file) + copyOrGenerateSymbolFile(executable_path, symbol_temp_dir, build_ffmpeg_log_file) copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file) # check that the copied file is runnable @@ -363,6 +385,10 @@ def main(): output = subprocess.check_output(args) build_ffmpeg_log_file.write(output.decode('utf-8')) + symbol_file_name = base_artifact_name + '-symbols' + shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) + shutil.rmtree(symbol_temp_dir) + # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), From 95ae6378fdaeedc85a9b8ab1c404e3daeb0bd954 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 22:11:39 -0600 Subject: [PATCH 19/23] Remove `--enable_small` for beamcoder --- build-ffmpeg | 1 - 1 file changed, 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index ced284fb..77903aee 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -1076,7 +1076,6 @@ if $SHARED_LIBRARIES; then --disable-static \ --enable-pthreads \ --enable-shared\ - --enable-small \ --enable-version3 \ --extra-cflags="${CFLAGS}" \ --extra-ldexeflags="${LDEXEFLAGS}" \ From c48d8e0b52fd452acfb3cfe92d19bebe4c00577b Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 15:16:42 -0600 Subject: [PATCH 20/23] Don't compile giflib in parallel It fails with too many cores --- build-ffmpeg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index 86ad1b3e..045c7e45 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -344,7 +344,10 @@ if build "giflib" "5.2.1"; then download "https://sourceforge.net/p/giflib/bugs/_discuss/thread/4e811ad29b/c323/attachment/Makefile.patch" execute patch "${PACKAGES}/giflib-5.2.1/Makefile" ${PACKAGES}/Makefile.patch"" fi - execute make -j $MJOBS + # Descript: compilation will fail with more than 8 jobs, so + # force to not run in parallel + #execute make -j $MJOBS + execute make execute make PREFIX="${WORKSPACE}" install build_done "giflib" "5.2.1" fi From ba0110e8a4fa90cad9d27d6d0176d0e65b33f50a Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 15:17:15 -0600 Subject: [PATCH 21/23] fail if build-ffmpeg fails --- descript/build-ffmpeg-descript.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 80bd0118..9a42c5da 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -59,8 +59,9 @@ def buildFFmpeg(script_dir, log_file): '-b', # build '--full-shared', # custom Descript shim to build shared libraries instead of static '--enable-gpl-and-free'] # custom Descript shim to build GPL but not non-free (libpostproc is needed by Beamcoder and requires GPL) - log_file.write(' '.join(args) + '\n\n') - subprocess.call(args, env=env, stdout=log_file) + log_file.write(' '.join(args) + '\n\n') + log_file.flush() + subprocess.run(args, env=env, stdout=log_file, stderr=log_file, check=True) # # From 797fbe7470d92d2d5b0aa894a8cd6f4aaa60657f Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 15:44:20 -0600 Subject: [PATCH 22/23] Fix relpath -> realpath typo --- descript/build-ffmpeg-descript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 9a42c5da..62fd3251 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -260,7 +260,7 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies - copyLibraryAndDependencies(os.path.relpath(src_dependency_file), dest_folder, log_file) + copyLibraryAndDependencies(os.path.realpath(src_dependency_file), dest_folder, log_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: From 499a002de8e8773d065d96e2943f9000bf0fb67c Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 17:00:53 -0600 Subject: [PATCH 23/23] Fix python 3.8 usage and better logging for missing libs --- descript/build-ffmpeg-descript.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 62fd3251..29770960 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -180,7 +180,7 @@ def getFileBaseNameWithoutVersion(file_path) -> str: # # # -def getVersionVariantsForFile(file_path) -> list[str]: +def getVersionVariantsForFile(file_path): """ Returns the following three files for any one of the file paths provided: `'.../ffmpeg-build-script/workspace/lib/libavcodec.58.134.100.dylib'` @@ -206,10 +206,11 @@ def getVersionVariantsForFile(file_path) -> list[str]: # # # -def copyLibraryAndDependencies(src_file, dest_folder, log_file): +def copyLibraryAndDependencies(src_file, dest_folder, log_file, parent_path = ''): """ Recursive function to copy a library and its (non-system) dependencies also fixes loader paths for each library to be `@loader_path` + :param: `parent_path` - optional argument to show which parent is linking against `src_file` """ dest_file = os.path.join(dest_folder, os.path.basename(src_file)) @@ -243,7 +244,7 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): if src_dependency_file.startswith('/usr/local'): # the build grabbed libraries installed on this machine # which might not be available on other machines - missing_libs.add(src_dependency_file) + missing_libs.add(f'{src_dependency_file} (dependency of {os.path.basename(parent_path)})') elif src_dependency_file.startswith(workspace_dir): dependency_name = os.path.basename(src_dependency_file) if not len(this_id): @@ -260,7 +261,11 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies - copyLibraryAndDependencies(os.path.realpath(src_dependency_file), dest_folder, log_file) + copyLibraryAndDependencies( + os.path.realpath(src_dependency_file), + dest_folder, + log_file, + src_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: @@ -376,7 +381,7 @@ def main(): build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) copyOrGenerateSymbolFile(executable_path, symbol_temp_dir, build_ffmpeg_log_file) - copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file) + copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file, executable_path) # check that the copied file is runnable build_ffmpeg_log_file.write('\nChecking ' + executable + '\n')