diff --git a/.github/actions/python-build/action.yml b/.github/actions/python-build/action.yml index f0370ebc6..fad19cd91 100644 --- a/.github/actions/python-build/action.yml +++ b/.github/actions/python-build/action.yml @@ -39,6 +39,12 @@ runs: PROJECT_NAME="stt" + OS=$(uname) + if [ "${OS}" = "Linux" -a "${{ inputs.target }}" != "host" ]; then + python3 -m venv stt-build + source stt-build/bin/activate + fi + NUMPY_BUILD_VERSION="${{ inputs.numpy_build }}" \ NUMPY_DEP_VERSION="${{ inputs.numpy_dep }}" \ EXTRA_CFLAGS=${{ inputs.local_cflags }} \ diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7f1468e41..2cadbd456 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -111,6 +111,48 @@ jobs: with: name: ${{ github.job }} path: ${{ github.workspace }}/build-static/ + libsox3_Linux: + name: "Lin|Build libsox3" + runs-on: ubuntu-20.04 + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} + steps: + - run: | + curl -sSL https://github.com/coqui-ai/STT/releases/download/v0.10.0-alpha.7/sox-14.4.2.tar.bz2 | tar xjf - + - run: | + mkdir -p sox-build/ + - uses: actions/cache@v2 + id: sox-build-cache + with: + path: sox-build/ + key: sox-1-${{ runner.os }}-${{ env.sox_hash }} + - name: Configure + run: | + cd sox-14.4.2 + ./configure \ + --with-dyn-default --enable-dl-sndfile --enable-dl-amrnb \ + --without-magic --without-png --without-ladspa --without-mad \ + --without-id3tag --without-lame --without-twolame \ + --without-libltdl --disable-openmp --disable-dependency-tracking \ + --with-gsm=no --with-lpc10=no --with-oss=no \ + --prefix=${{ github.workspace }}/sox-build/ + if: steps.sox-build-cache.outputs.cache-hit != 'true' + - name: Make + run: | + cd sox-14.4.2 + make -j + if: steps.sox-build-cache.outputs.cache-hit != 'true' + - name: Make install + run: | + cd sox-14.4.2 + make install + if: steps.sox-build-cache.outputs.cache-hit != 'true' + - uses: actions/upload-artifact@v2 + with: + name: ${{ github.job }} + path: ${{ github.workspace }}/sox-build/ build-ctc-decoder-Linux: name: "Lin|Build CTC decoder Python package" needs: [ swig_Linux ] @@ -286,7 +328,7 @@ jobs: build-lib_Linux: name: "Lin|Build libstt+client" runs-on: ubuntu-20.04 - needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ] + needs: [build-tensorflow-Linux, tensorflow_opt-Linux, libsox3_Linux] container: image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c volumes: @@ -303,9 +345,7 @@ jobs: - name: Install dependencies run: | apt-get update - apt-get install -y --no-install-recommends \ - xz-utils libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev \ - zlib1g-dev zip + apt-get install -y --no-install-recommends xz-utils zip - run: | tar --skip-old-files -xf ${{ needs.tensorflow_opt-linux.outputs.cache_key }}.tar.xz rm ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz @@ -313,6 +353,10 @@ jobs: run: | /opt/python/cp37-cp37m/bin/python -m venv /tmp/venv echo "/tmp/venv/bin" >> $GITHUB_PATH + - uses: actions/download-artifact@v2 + with: + name: "libsox3_Linux" + path: ${{ github.workspace }}/sox-build/ - uses: ./.github/actions/host-build - uses: ./.github/actions/package - uses: actions/upload-artifact@v2 @@ -326,7 +370,7 @@ jobs: build-python-Linux: name: "Lin|Build Python bindings" runs-on: ubuntu-20.04 - needs: [ build-lib_Linux, swig_Linux ] + needs: [build-lib_Linux, swig_Linux] strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] @@ -527,6 +571,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -580,6 +625,7 @@ jobs: electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1113,7 +1159,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-macOS.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-macOS: name: "Mac|Build NodeJS and ElectronJS" runs-on: macos-10.15 @@ -1260,6 +1306,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1571,7 +1618,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-Windows.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-Windows: name: "Win|Build NodeJS/ElectronJS" runs-on: windows-2019 @@ -1747,6 +1794,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1922,6 +1970,7 @@ jobs: nodejs-version: [10, 16] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1975,6 +2024,7 @@ jobs: electronjs-version: [5.0.13, 12.0.0] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2030,6 +2080,7 @@ jobs: nodejs-version: [10, 16] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2081,6 +2132,7 @@ jobs: electronjs-version: [5.0.13, 12.0.0] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2134,6 +2186,7 @@ jobs: nodejs-version: [10, 16] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2197,6 +2250,7 @@ jobs: electronjs-version: [5.0.13, 12.0.0] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2421,7 +2475,7 @@ jobs: name: "libstt.tflite.linux.aarch64.zip" path: ${{ github.workspace }}/artifacts/libstt.zip build-python-LinuxArmv7: - name: "LinArmv7|Build python bindings" + name: "LinArmv7|Build Python bindings" runs-on: ubuntu-20.04 needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ] strategy: @@ -2477,6 +2531,9 @@ jobs: uses: ./.github/actions/numpy_vers with: pyver: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install -U pip setuptools wheel - uses: ./.github/actions/python-build with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" @@ -2486,7 +2543,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-armv7.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-LinuxArmv7: name: "LinArmv7|Build NodeJS and ElectronJS" runs-on: ubuntu-20.04 @@ -2561,7 +2618,7 @@ jobs: name: "stt_intermediate-tflite-armv7.tgz" path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz build-python-LinuxAarch64: - name: "LinAarch64|Build python bindings" + name: "LinAarch64|Build Python bindings" runs-on: ubuntu-20.04 needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ] strategy: @@ -2619,6 +2676,9 @@ jobs: uses: ./.github/actions/numpy_vers with: pyver: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install -U pip setuptools wheel - uses: ./.github/actions/python-build with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" @@ -2628,7 +2688,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-aarch64.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-LinuxAarch64: name: "LinAarch64|Build NodeJS and ElectronJS" runs-on: ubuntu-20.04 @@ -2864,6 +2924,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" diff --git a/ci_scripts/package-utils.sh b/ci_scripts/package-utils.sh index c59e6f8cd..088b28d12 100755 --- a/ci_scripts/package-utils.sh +++ b/ci_scripts/package-utils.sh @@ -34,12 +34,18 @@ package_native_client() win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib" fi; + libsox_lib="" + if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then + libsox_lib="-C ${stt_dir}/sox-build/lib libsox.so.3" + fi + ${TAR} --verbose -cf - \ --transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \ -C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \ -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \ ${win_lib} \ + ${libsox_lib} \ -C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \ -C ${stt_dir}/ LICENSE \ -C ${stt_dir}/native_client/ stt${PLATFORM_EXE_SUFFIX} \ @@ -85,6 +91,7 @@ package_native_client_ndk() package_libstt_as_zip() { tensorflow_dir=${DS_TFDIR} + stt_dir=${DS_DSDIR} artifacts_dir=${CI_ARTIFACTS_DIR} artifact_name=$1 @@ -99,8 +106,14 @@ package_libstt_as_zip() echo "Please specify artifact name." fi; + libsox_lib="" + if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then + libsox_lib="${stt_dir}/sox-build/lib/libsox.so.3" + fi + ${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \ ${tensorflow_dir}/bazel-bin/native_client/libstt.so \ ${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \ + ${libsox_lib} \ ${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so } diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 7aab70f2e..1f55b83b2 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -1,6 +1,7 @@ NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) TARGET ?= host +ROOT_DIR ?= $(abspath $(NC_DIR)/..) TFDIR ?= $(abspath $(NC_DIR)/../tensorflow) PREFIX ?= /usr/local SO_SEARCH ?= $(TFDIR)/bazel-bin/ @@ -28,7 +29,7 @@ TOOLCHAIN := CFLAGS := CXXFLAGS := LDFLAGS := -SOX_CFLAGS := `pkg-config --cflags sox` +SOX_CFLAGS := -I$(ROOT_DIR)/sox-build/include ifeq ($(OS),Linux) MAGIC_LINK_LZMA := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.so | grep lzma | grep '*UND*' | wc -l) ifneq ($(MAGIC_LINK_LZMA),0) @@ -38,8 +39,7 @@ MAGIC_LINK_BZ2 := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.s ifneq ($(MAGIC_LINK_BZ2),0) MAYBE_LINK_BZ2 := -lbz2 endif # MAGIC_LINK_BZ2 -SOX_CFLAGS += -fopenmp -SOX_LDFLAGS := `pkg-config --libs sox` -lgsm `pkg-config --libs libpng | cut -d' ' -f1` -lz -lmagic $(MAYBE_LINK_LZMA) $(MAYBE_LINK_BZ2) -lltdl -ldl +SOX_LDFLAGS := -L$(ROOT_DIR)/sox-build/lib -lsox else ifeq ($(OS),Darwin) LIBSOX_PATH := $(shell echo `pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib) LIBOPUSFILE_PATH := $(shell echo `pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib) @@ -175,7 +175,7 @@ define copy_missing_libs SRC_FILE=$(1); \ TARGET_LIB_DIR=$(2); \ MANIFEST_IN=$(3); \ - echo "Analyzing $$SRC_FILE copying missing libs to $$SRC_FILE"; \ + echo "Analyzing $$SRC_FILE copying missing libs to $$TARGET_LIB_DIR"; \ echo "Maybe outputting to $$MANIFEST_IN"; \ \ (mkdir $$TARGET_LIB_DIR || true); \ @@ -191,6 +191,7 @@ define copy_missing_libs fi; \ done; \ \ + echo "Missing libs = $$missing_libs"; \ for missing in $$missing_libs; do \ find $(SO_SEARCH) -type f -name "$$missing" -exec cp {} $$TARGET_LIB_DIR \; ; \ chmod +w $$TARGET_LIB_DIR/*.so ; \