Enhance cross-platform build system (#266)

With this new build system, it'll make it easier to compile third-party
libraries statically to multiple platforms.

.gitignore

@@ -10,3 +10,5 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+scripts/cross

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "cvendor/src/openh264"]
-	path = cvendor/src/openh264
-	url = https://github.com/cisco/openh264.git

Makefile

@@ -1,56 +0,0 @@
-vendor_dir = cvendor
-src_dir = $(vendor_dir)/src
-lib_dir = $(vendor_dir)/lib
-include_dir = $(vendor_dir)/include
-
-make_args.x86_64-windows = \
-	CC=x86_64-w64-mingw32-gcc \
-	CXX=x86_64-w64-mingw32-g++ \
-	ARCH=x86_64 \
-	OS=mingw_nt
-make_args.x86_64-darwin = \
-	CC=o64-clang \
-	CXX=o64-clang++ \
-	AR=llvm-ar \
-	ARCH=x86_64 \
-	OS=darwin
-
-.PHONY: vendor
-vendor: \
-	$(include_dir)/openh264 \
-	cross-libraries
-
-$(include_dir)/openh264: $(src_dir)/openh264
-	mkdir -p $@
-	cp $^/codec/api/svc/*.h $@
-
-$(lib_dir)/openh264/libopenh264.x86_64-linux.a: $(src_dir)/openh264
-	$(MAKE) -C $^ clean \
-		&& $(MAKE) -C $^ libraries
-	mkdir -p $(dir $@)
-	cp $^/libopenh264.a $@
-
-$(lib_dir)/openh264/libopenh264.x86_64-windows.a: $(src_dir)/openh264
-	$(MAKE) -C $^ clean \
-		&& $(MAKE) -C $^ $(make_args.x86_64-windows) libraries
-	mkdir -p $(dir $@)
-	cp $^/libopenh264.a $@
-
-$(lib_dir)/openh264/libopenh264.x86_64-darwin.a: $(src_dir)/openh264
-	$(MAKE) -C $^ clean \
-		&& $(MAKE) -C $^ $(make_args.x86_64-darwin) libraries
-	mkdir -p $(dir $@)
-	cp $^/libopenh264.a $@
-
-.PHONY: cross-libraries
-cross-libraries:
-	docker build -t mediadevices-libs-builder -f libs-builder.Dockerfile .
-	docker run --rm \
-		-v $(CURDIR):/go/src/github.com/pion/mediadevices \
-		mediadevices-libs-builder make $(lib_dir)/openh264/libopenh264.x86_64-linux.a
-	docker run --rm \
-		-v $(CURDIR):/go/src/github.com/pion/mediadevices \
-		mediadevices-libs-builder make $(lib_dir)/openh264/libopenh264.x86_64-windows.a
-	docker run --rm \
-		-v $(CURDIR):/go/src/github.com/pion/mediadevices \
-		mediadevices-libs-builder make $(lib_dir)/openh264/libopenh264.x86_64-darwin.a

build.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+MEDIADEVICES_TOOLCHAIN_OWNER=lherman
+MEDIADEVICES_TOOLCHAIN_PREFIX=cross
+MEDIADEVICES_SCRIPT_PATH=$(realpath ./scripts)
+MEDIADEVICES_TOOLCHAIN_PATH=${MEDIADEVICES_SCRIPT_PATH}/${MEDIADEVICES_TOOLCHAIN_PREFIX}
+MEDIADEVICES_DOCKERFILES_PATH=dockerfiles
+
+# Reference: https://github.com/dockcross/dockcross#cross-compilers
+MEDIADEVICES_TARGET_PLATFORMS=(
+  linux-armv7
+  linux-arm64
+  linux-x64
+  windows-x64
+  darwin-x64
+)
+
+if [[ -z ${VERBOSE} ]]; then
+  MEDIADEVICES_OUTPUT=/dev/null
+else
+  MEDIADEVICES_OUTPUT=/dev/stdout
+fi
+
+install_toolchains() {
+  bash ${MEDIADEVICES_DOCKERFILES_PATH}/build.sh &> ${MEDIADEVICES_OUTPUT}
+  for platform in ${MEDIADEVICES_TARGET_PLATFORMS[@]}
+  do
+    mkdir -p ${MEDIADEVICES_TOOLCHAIN_PATH}
+    image=${MEDIADEVICES_TOOLCHAIN_OWNER}/${MEDIADEVICES_TOOLCHAIN_PREFIX}-${platform}
+    bin_path=${MEDIADEVICES_TOOLCHAIN_PATH}/${MEDIADEVICES_TOOLCHAIN_PREFIX}-${platform}
+    docker run ${image} > ${bin_path}
+    chmod +x ${bin_path}
+  done
+}
+
+build() {
+  sub_builds=$(find pkg -type f -name "build.sh")
+  for sub_build in ${sub_builds[@]}
+  do
+    sub_build=$(realpath ${sub_build})
+    sub_build_dir=$(dirname ${sub_build})
+    current_dir=${PWD}
+    cd $sub_build_dir
+    for platform in ${MEDIADEVICES_TARGET_PLATFORMS[@]}
+    do
+      export MEDIADEVICES_TOOLCHAIN_BIN=${MEDIADEVICES_TOOLCHAIN_PATH}/${MEDIADEVICES_TOOLCHAIN_PREFIX}-${platform}
+      # convert '-' to '_' since '_' is more common in library names
+      export MEDIADEVICES_TARGET_PLATFORM=${platform//-/_}
+      export MEDIADEVICES_TARGET_OS=$(echo $MEDIADEVICES_TARGET_PLATFORM | cut -d'_' -f1)
+      export MEDIADEVICES_TARGET_ARCH=${platform//${MEDIADEVICES_TARGET_OS}-/}
+
+      echo "Building ${sub_build_dir}:"
+      echo "  PLATFORM      : ${MEDIADEVICES_TARGET_PLATFORM}"
+      echo "  OS            : ${MEDIADEVICES_TARGET_OS}"
+      echo "  ARCH          : ${MEDIADEVICES_TARGET_ARCH}"
+      echo "  TOOLCHAIN_BIN : ${MEDIADEVICES_TOOLCHAIN_BIN}"
+      echo ""
+
+      ${sub_build} &> ${MEDIADEVICES_OUTPUT}
+    done
+    cd ${current_dir}
+  done
+}
+
+if [[ $# > 0 && $1 != "all" ]]; then
+  MEDIADEVICES_TARGET_PLATFORMS=($1)
+fi
+
+install_toolchains
+build

dockerfiles/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+OWNER=lherman
+PREFIX=cross
+IMAGES=$(ls *.Dockerfile)
+
+for image in ${IMAGES[@]}
+do
+  tag=${OWNER}/cross-${image//.Dockerfile/}
+  docker build -t "${tag}" -f "$image" .
+done

dockerfiles/darwin-x64.Dockerfile

@@ -0,0 +1,20 @@
+FROM dockcross/base
+
+ENV OSX_CROSS_PATH=/osxcross
+
+COPY --from=dockercore/golang-cross "${OSX_CROSS_PATH}/." "${OSX_CROSS_PATH}/"
+ENV PATH=${OSX_CROSS_PATH}/target/bin:$PATH
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+
+ENV CC=x86_64-apple-darwin14-clang \
+    CXX=x86_64-apple-darwin14-clang++ \
+    CPP=x86_64-apple-darwin14-clang++ \
+    AR=x86_64-apple-darwin14-ar \
+    AS=x86_64-apple-darwin14-as \
+    LD=x86_64-apple-darwin14-ld
+
+ARG IMAGE=lherman/cross-darwin-x64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}

dockerfiles/init.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+apt-get update 
+apt-get install -y nasm clang llvm
+
+curl -L https://golang.org/dl/go1.15.6.linux-amd64.tar.gz | tar -C /usr/local -xzf -
+ln -s /usr/local/go/bin/go /usr/local/bin/go

dockerfiles/linux-arm64.Dockerfile

@@ -0,0 +1,8 @@
+FROM dockcross/linux-arm64
+
+ARG IMAGE=lherman/cross-linux-arm64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh

dockerfiles/linux-armv7.Dockerfile

@@ -0,0 +1,8 @@
+FROM dockcross/linux-armv7
+
+ARG IMAGE=lherman/cross-linux-armv7
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh

dockerfiles/linux-x64.Dockerfile

@@ -0,0 +1,8 @@
+FROM dockcross/linux-x64
+
+ARG IMAGE=lherman/cross-linux-x64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh

dockerfiles/windows-x64.Dockerfile

@@ -0,0 +1,11 @@
+FROM dockcross/windows-static-x64-posix
+
+ARG IMAGE=lherman/cross-windows-x64
+ARG VERSION=latest
+ENV DEFAULT_DOCKCROSS_IMAGE ${IMAGE}:${VERSION}
+
+COPY init.sh /tmp/init.sh
+RUN bash /tmp/init.sh
+RUN ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static.posix-gcc /usr/bin/x86_64-w64-mingw32-gcc && \
+    ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static.posix-g++ /usr/bin/x86_64-w64-mingw32-g++ && \
+    ln -s /usr/src/mxe/usr/bin/x86_64-w64-mingw32.static.posix-ar /usr/bin/x86_64-w64-mingw32-ar

libs-builder.Dockerfile

@@ -1,10 +0,0 @@
-FROM dockercore/golang-cross
-
-RUN apt-get update -qq \
-  && apt-get install -y \
-    g++-mingw-w64 \
-    nasm \
-  && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-VOLUME /go/src/github.com/pion/mediadevices
-WORKDIR /go/src/github.com/pion/mediadevices

pkg/codec/openh264/.gitignore

@@ -0,0 +1 @@
+src

pkg/codec/openh264/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2013, Cisco Systems
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/codec/openh264/build.sh

@@ -0,0 +1,36 @@
+GIT_URL=https://github.com/cisco/openh264.git
+VERSION=v2.1.1
+SRC_DIR=src
+LIB_DIR=lib
+INCLUDE_DIR=include/openh264
+ROOT_DIR=${PWD}
+LIB_PREFIX=libopenh264
+
+OS=${MEDIADEVICES_TARGET_OS}
+ARCH=${MEDIADEVICES_TARGET_ARCH}
+
+case ${MEDIADEVICES_TARGET_OS} in
+  windows)
+    OS=mingw_nt
+    ;;
+esac
+
+case ${MEDIADEVICES_TARGET_ARCH} in
+  armv6 | armv7 | armv8)
+    ARCH=arm
+    ;;
+  x64)
+    ARCH=x86_64
+    ;;
+esac
+
+mkdir -p ${LIB_DIR} ${INCLUDE_DIR}
+
+git clone --depth=1 --branch=${VERSION} ${GIT_URL} ${SRC_DIR}
+cd ${SRC_DIR}
+${MEDIADEVICES_TOOLCHAIN_BIN} make -j2 ${LIB_PREFIX}.a OS=${OS} ARCH=${ARCH}
+${MEDIADEVICES_TOOLCHAIN_BIN} echo $PATH
+mv ${LIB_PREFIX}.a ${ROOT_DIR}/${LIB_DIR}/${LIB_PREFIX}_${MEDIADEVICES_TARGET_PLATFORM}.a
+mkdir -p ${ROOT_DIR}/${INCLUDE_DIR}
+cp codec/api/svc/*.h ${ROOT_DIR}/${INCLUDE_DIR}
+make clean

pkg/codec/openh264/include/openh264/codec_app_def.h

@@ -167,8 +167,8 @@ typedef enum {
   DECODER_OPTION_LEVEL,                 ///< get current AU level info,only is used in GetOption
   DECODER_OPTION_STATISTICS_LOG_INTERVAL,///< set log output interval
   DECODER_OPTION_IS_REF_PIC,             ///< feedback current frame is ref pic or not
-  DECODER_OPTION_NUM_OF_FRAMES_REMAINING_IN_BUFFER  ///< number of frames remaining in decoder buffer when pictures are required to re-ordered into display-order.
+  DECODER_OPTION_NUM_OF_FRAMES_REMAINING_IN_BUFFER,  ///< number of frames remaining in decoder buffer when pictures are required to re-ordered into display-order.
-
+  DECODER_OPTION_NUM_OF_THREADS,         ///< number of decoding threads. The maximum thread count is equal or less than lesser of (cpu core counts and 16).
 } DECODER_OPTION;
 
 /**

pkg/codec/openh264/include/openh264/codec_def.h

@@ -201,6 +201,7 @@ typedef struct TagBufferInfo {
   union {
     SSysMEMBuffer sSystemBuffer; ///<  memory info for one picture
   } UsrData;                     ///<  output buffer info
+  unsigned char* pDst[3];  //point to picture YUV data
 } SBufferInfo;
 
 

pkg/codec/openh264/include/openh264/codec_ver.h

@@ -4,12 +4,12 @@
 
 #include "codec_app_def.h"
 
-static const OpenH264Version g_stCodecVersion  = {2, 0, 0, 1905};
+static const OpenH264Version g_stCodecVersion  = {2, 1, 0, 2002};
-static const char* const g_strCodecVer  = "OpenH264 version:2.0.0.1905";
+static const char* const g_strCodecVer  = "OpenH264 version:2.1.0.2002";
 
 #define OPENH264_MAJOR (2)
-#define OPENH264_MINOR (0)
+#define OPENH264_MINOR (1)
 #define OPENH264_REVISION (0)
-#define OPENH264_RESERVED (1905)
+#define OPENH264_RESERVED (2002)
 
 #endif  // CODEC_VER_H

pkg/codec/openh264/openh264.go

@@ -1,7 +1,5 @@
 package openh264
 
-// #cgo CFLAGS: -I${SRCDIR}/../../../cvendor/include
-// #cgo CXXFLAGS: -I${SRCDIR}/../../../cvendor/include
 // #include <string.h>
 // #include <openh264/codec_api.h>
 // #include <errno.h>

pkg/codec/openh264/openh264_darwin.go

@@ -1,6 +0,0 @@
-// +build !dynamic
-
-package openh264
-
-// #cgo LDFLAGS: ${SRCDIR}/../../../cvendor/lib/openh264/libopenh264.x86_64-darwin.a
-import "C"

pkg/codec/openh264/openh264_linux.go

@@ -1,6 +0,0 @@
-// +build !dynamic
-
-package openh264
-
-// #cgo LDFLAGS: ${SRCDIR}/../../../cvendor/lib/openh264/libopenh264.x86_64-linux.a
-import "C"

pkg/codec/openh264/openh264_static.go

@@ -0,0 +1,12 @@
+// +build !dynamic
+
+package openh264
+
+//#cgo CFLAGS: -I${SRCDIR}/include
+//#cgo CXXFLAGS: -I${SRCDIR}/include
+//#cgo linux,arm LDFLAGS: ${SRCDIR}/lib/libopenh264_linux_armv7.a
+//#cgo linux,arm64 LDFLAGS: ${SRCDIR}/lib/libopenh264_linux_arm64.a
+//#cgo linux,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264_linux_x64.a
+//#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264_darwin_x64.a
+//#cgo windows,amd64 LDFLAGS: ${SRCDIR}/lib/libopenh264_windows_x64.a -lssp
+import "C"

pkg/codec/openh264/openh264_windows.go

@@ -1,14 +0,0 @@
-// +build !dynamic
-
-package openh264
-
-// #cgo LDFLAGS: ${SRCDIR}/../../../cvendor/lib/openh264/libopenh264.x86_64-windows.a -lssp
-// #include <stdio.h>
-// #include <crtdefs.h>
-// #if __MINGW64_VERSION_MAJOR < 6
-// // Workaround for the breaking ABI change between MinGW 5 and 6
-//   FILE *__cdecl __acrt_iob_func(unsigned index) { return &(__iob_func()[index]); }
-//   typedef FILE *__cdecl (*_f__acrt_iob_func)(unsigned index);
-//   _f__acrt_iob_func __MINGW_IMP_SYMBOL(__acrt_iob_func) = __acrt_iob_func;
-// #endif
-import "C"

scripts/install-dependencies

@@ -1,117 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import subprocess
-import shutil
-import sys
-
-TOOLCHAINS = {
-    'windows': 'x86_64-w64-mingw32'
-}
-
-TARGET_OS = os.environ.get('GOOS', '')
-VALID_TARGETS = TOOLCHAINS.keys()
-if TARGET_OS not in VALID_TARGETS:
-    sys.exit(f'{TARGET_OS} is not in supported targets: {",".join(VALID_TARGETS)}')
-
-TOOLCHAIN = TOOLCHAINS[TARGET_OS]
-
-'''
-Format:
-
-Git Dependency:
-name: <dependency_name>
-url: <repository_url>
-version: <git_tag>
-protocol: 'git'
-cmd: <command_how_to_install>
-
-HTTP Dependency:
-name: <dependency_name>
-url: <http_url>
-extension: <file_extension_that_will_be_downloaded>
-protocol: 'http'
-cmd: <command_how_to_install>
-'''
-DEPENDENCIES = []
-ROOT_DIR = '/tmp'
-
-
-def log(name, description):
-    print(f'{name}: {description}')
-
-
-def run_cmd(command):
-    p = subprocess.run(command, shell=True, capture_output=True)
-
-    # If a command fails, exit with the return code with the error message.
-    if p.returncode != 0:
-        sys.exit(p.stderr.decode('utf-8'))
-
-    return p
-
-
-def pull_from_git(dependency):
-    name = dependency['name']
-    url = dependency['url']
-    version = dependency['version']
-
-    run_cmd(f'git clone --branch={version} --depth=1 {url} {name}')
-    install(dependency)
-
-
-def pull_from_http(dependency):
-    name = dependency['name']
-    url = dependency['url']
-    extension = dependency['extension']
-    tar_name = f'{name}.{extension}'
-
-    run_cmd(f'curl -L {url} > {tar_name}')
-    shutil.unpack_archive(tar_name, name)
-
-    extracted = os.listdir(name)
-
-    # If it's being wrapped with another folder, we move them
-    # one level higher so that we have all the source codes in the root
-    if len(extracted) == 1:
-        pwd = os.getcwd()
-        package_folder = os.path.join(pwd, name)
-        wrapper_folder = os.path.join(package_folder, extracted[0])
-        if os.path.isdir(wrapper_folder):
-            for to_move in os.listdir(wrapper_folder):
-                src = os.path.join(name, wrapper_folder, to_move)
-                dst = os.path.join(package_folder, to_move)
-                shutil.move(src, dst)
-
-    install(dependency)
-
-
-def pull(dependency):
-    protocol = dependency['protocol']
-    name = dependency['name']
-    handlers = {
-        'http': pull_from_http,
-        'git': pull_from_git,
-    }
-
-    log(name, f'pulling with {protocol} protocol')
-    return handlers[protocol](dependency)
-
-
-def install(dependency):
-    name = dependency['name']
-    cmd = dependency['cmd']
-
-    log(name, f'installing')
-    os.chdir(os.path.join(ROOT_DIR, name))
-    run_cmd(cmd)
-
-
-os.chdir(ROOT_DIR)
-for dependency in DEPENDENCIES:
-    pull(dependency)
-
-print()
-print('installed dependencies:')
-p = run_cmd('pkg-config --list-all')
-print(p.stdout.decode('utf-8'))

windows.Dockerfile

@@ -1,33 +0,0 @@
-FROM fedora
-
-ENV PYTHONUNBUFFERED=1 \
-    # Somehow libopus.dll is installed under /usr/local/bin, so we need to add this
-    # path to WINEPATH so that wine can find this library
-    WINEPATH=/usr/local/bin \
-    # Go and C default environments
-    GOOS=windows \
-    GOARCH=amd64 \
-    CGO_ENABLED=1 \
-    GO111MODULE=on \
-    CC=x86_64-w64-mingw32-gcc \
-    CXX=x86_64-w64-mingw32-g++
-
-COPY scripts /usr/bin/ 
-
-RUN dnf install -y \
-        diffutils \
-        golang \
-        make \
-        mingw64-gcc \
-        mingw64-gcc-c++ \
-        mingw64-opus && \
-        nasm \
-        pkg-config \
-        python3 \
-        wine \
-    dnf clean all && \
-    install-dependencies && \
-    rm -rf /tmp/*
-
-VOLUME /go/src/github.com/pion/mediadevices
-WORKDIR /go/src/github.com/pion/mediadevices