fixup

coqui-ai · Aug 1, 2021 · d33ba0b · d33ba0b
1 parent 66187e8
commit d33ba0b
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 15 deletions.
diff --git a/.github/actions/python-build/action.yml b/.github/actions/python-build/action.yml
@@ -39,6 +39,12 @@ runs:
 
         PROJECT_NAME="stt"
 
+        OS=$(uname)
+        if [ "${OS}" = "Linux" -a "${{ inputs.target }}" != "host" ]; then
+          python3 -m venv stt-build
+          source stt-build/bin/activate
+        fi
+
         NUMPY_BUILD_VERSION="${{ inputs.numpy_build }}" \
         NUMPY_DEP_VERSION="${{ inputs.numpy_dep }}" \
         EXTRA_CFLAGS=${{ inputs.local_cflags }} \

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -111,6 +111,48 @@ jobs:
         with:
           name: ${{ github.job }}
           path: ${{ github.workspace }}/build-static/
+  libsox3_Linux:
+    name: "Lin|Build libsox3"
+    runs-on: ubuntu-20.04
+    container:
+      image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
+      volumes:
+        - ${{ github.workspace }}:${{ github.workspace }}
+    steps:
+      - run: |
+          curl -sSL https://github.com/coqui-ai/STT/releases/download/v0.10.0-alpha.7/sox-14.4.2.tar.bz2 | tar xjf -
+      - run: |
+          mkdir -p sox-build/
+      - uses: actions/cache@v2
+        id: sox-build-cache
+        with:
+          path: sox-build/
+          key: sox-1-${{ runner.os }}-${{ env.sox_hash }}
+      - name: Configure
+        run: |
+          cd sox-14.4.2
+          ./configure \
+            --with-dyn-default --enable-dl-sndfile --enable-dl-amrnb \
+            --without-magic --without-png --without-ladspa --without-mad \
+            --without-id3tag --without-lame --without-twolame \
+            --without-libltdl --disable-openmp --disable-dependency-tracking \
+            --with-gsm=no --with-lpc10=no --with-oss=no \
+            --prefix=${{ github.workspace }}/sox-build/
+        if: steps.sox-build-cache.outputs.cache-hit != 'true'
+      - name: Make
+        run: |
+          cd sox-14.4.2
+          make -j
+        if: steps.sox-build-cache.outputs.cache-hit != 'true'
+      - name: Make install
+        run: |
+          cd sox-14.4.2
+          make install
+        if: steps.sox-build-cache.outputs.cache-hit != 'true'
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}
+          path: ${{ github.workspace }}/sox-build/
   build-ctc-decoder-Linux:
     name: "Lin|Build CTC decoder Python package"
     needs: [ swig_Linux ]
@@ -286,7 +328,7 @@ jobs:
   build-lib_Linux:
     name: "Lin|Build libstt+client"
     runs-on: ubuntu-20.04
-    needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ]
+    needs: [build-tensorflow-Linux, tensorflow_opt-Linux, libsox3_Linux]
     container:
       image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
       volumes:
@@ -303,16 +345,18 @@ jobs:
       - name: Install dependencies
         run: |
           apt-get update
-          apt-get install -y --no-install-recommends \
-            xz-utils libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev \
-            zlib1g-dev zip
+          apt-get install -y --no-install-recommends xz-utils zip
       - run: |
           tar --skip-old-files -xf ${{ needs.tensorflow_opt-linux.outputs.cache_key }}.tar.xz
           rm ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
       - name: Setup venv
         run: |
           /opt/python/cp37-cp37m/bin/python -m venv /tmp/venv
           echo "/tmp/venv/bin" >> $GITHUB_PATH
+      - uses: actions/download-artifact@v2
+        with:
+          name: "libsox3_Linux"
+          path: ${{ github.workspace }}/sox-build/
       - uses: ./.github/actions/host-build
       - uses: ./.github/actions/package
       - uses: actions/upload-artifact@v2
@@ -326,7 +370,7 @@ jobs:
   build-python-Linux:
     name: "Lin|Build Python bindings"
     runs-on: ubuntu-20.04
-    needs: [ build-lib_Linux, swig_Linux ]
+    needs: [build-lib_Linux, swig_Linux]
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
@@ -527,6 +571,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -580,6 +625,7 @@ jobs:
         electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1113,7 +1159,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-macOS.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-macOS:
     name: "Mac|Build NodeJS and ElectronJS"
     runs-on: macos-10.15
@@ -1260,6 +1306,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1571,7 +1618,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-Windows.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-Windows:
     name: "Win|Build NodeJS/ElectronJS"
     runs-on: windows-2019
@@ -1747,6 +1794,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1922,6 +1970,7 @@ jobs:
         nodejs-version: [10, 16]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1975,6 +2024,7 @@ jobs:
         electronjs-version: [5.0.13, 12.0.0]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2030,6 +2080,7 @@ jobs:
         nodejs-version: [10, 16]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2081,6 +2132,7 @@ jobs:
         electronjs-version: [5.0.13, 12.0.0]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2134,6 +2186,7 @@ jobs:
         nodejs-version: [10, 16]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2197,6 +2250,7 @@ jobs:
         electronjs-version: [5.0.13, 12.0.0]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2421,7 +2475,7 @@ jobs:
           name: "libstt.tflite.linux.aarch64.zip"
           path: ${{ github.workspace }}/artifacts/libstt.zip
   build-python-LinuxArmv7:
-    name: "LinArmv7|Build python bindings"
+    name: "LinArmv7|Build Python bindings"
     runs-on: ubuntu-20.04
     needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ]
     strategy:
@@ -2477,6 +2531,9 @@ jobs:
         uses: ./.github/actions/numpy_vers
         with:
           pyver: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -U pip setuptools wheel
       - uses: ./.github/actions/python-build
         with:
           numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
@@ -2486,7 +2543,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-armv7.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-LinuxArmv7:
     name: "LinArmv7|Build NodeJS and ElectronJS"
     runs-on: ubuntu-20.04
@@ -2561,7 +2618,7 @@ jobs:
           name: "stt_intermediate-tflite-armv7.tgz"
           path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
   build-python-LinuxAarch64:
-    name: "LinAarch64|Build python bindings"
+    name: "LinAarch64|Build Python bindings"
     runs-on: ubuntu-20.04
     needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ]
     strategy:
@@ -2619,6 +2676,9 @@ jobs:
         uses: ./.github/actions/numpy_vers
         with:
           pyver: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -U pip setuptools wheel
       - uses: ./.github/actions/python-build
         with:
           numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
@@ -2628,7 +2688,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-aarch64.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-LinuxAarch64:
     name: "LinAarch64|Build NodeJS and ElectronJS"
     runs-on: ubuntu-20.04
@@ -2864,6 +2924,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"

diff --git a/ci_scripts/package-utils.sh b/ci_scripts/package-utils.sh
@@ -34,12 +34,18 @@ package_native_client()
     win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib"
   fi;
 
+  libsox_lib=""
+  if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then
+    libsox_lib="-C ${stt_dir}/sox-build/lib libsox.so.3"
+  fi
+
   ${TAR} --verbose -cf - \
     --transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \
     -C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \
     -C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \
     -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \
     ${win_lib} \
+    ${libsox_lib} \
     -C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
     -C ${stt_dir}/ LICENSE \
     -C ${stt_dir}/native_client/ stt${PLATFORM_EXE_SUFFIX} \
@@ -85,6 +91,7 @@ package_native_client_ndk()
 package_libstt_as_zip()
 {
   tensorflow_dir=${DS_TFDIR}
+  stt_dir=${DS_DSDIR}
   artifacts_dir=${CI_ARTIFACTS_DIR}
   artifact_name=$1
 
@@ -99,8 +106,14 @@ package_libstt_as_zip()
     echo "Please specify artifact name."
   fi;
 
+  libsox_lib=""
+  if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then
+    libsox_lib="${stt_dir}/sox-build/lib/libsox.so.3"
+  fi
+
   ${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \
     ${tensorflow_dir}/bazel-bin/native_client/libstt.so \
     ${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \
+    ${libsox_lib} \
     ${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so
 }
diff --git a/native_client/definitions.mk b/native_client/definitions.mk
@@ -1,6 +1,7 @@
 NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 TARGET    ?= host
+ROOT_DIR  ?= $(abspath $(NC_DIR)/..)
 TFDIR     ?= $(abspath $(NC_DIR)/../tensorflow)
 PREFIX    ?= /usr/local
 SO_SEARCH ?= $(TFDIR)/bazel-bin/
@@ -28,7 +29,7 @@ TOOLCHAIN       :=
 CFLAGS          :=
 CXXFLAGS        :=
 LDFLAGS         :=
-SOX_CFLAGS      := `pkg-config --cflags sox`
+SOX_CFLAGS      := -I$(ROOT_DIR)/sox-build/include
 ifeq ($(OS),Linux)
 MAGIC_LINK_LZMA := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.so | grep lzma | grep '*UND*' | wc -l)
 ifneq ($(MAGIC_LINK_LZMA),0)
@@ -38,8 +39,7 @@ MAGIC_LINK_BZ2  := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.s
 ifneq ($(MAGIC_LINK_BZ2),0)
 MAYBE_LINK_BZ2  := -lbz2
 endif # MAGIC_LINK_BZ2
-SOX_CFLAGS      += -fopenmp
-SOX_LDFLAGS     := `pkg-config --libs sox` -lgsm `pkg-config --libs libpng | cut -d' ' -f1` -lz -lmagic $(MAYBE_LINK_LZMA) $(MAYBE_LINK_BZ2) -lltdl -ldl
+SOX_LDFLAGS     := -L$(ROOT_DIR)/sox-build/lib -lsox
 else ifeq ($(OS),Darwin)
 LIBSOX_PATH             := $(shell echo `pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib)
 LIBOPUSFILE_PATH        := $(shell echo `pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib)
@@ -175,7 +175,7 @@ define copy_missing_libs
     SRC_FILE=$(1); \
     TARGET_LIB_DIR=$(2); \
     MANIFEST_IN=$(3); \
-    echo "Analyzing $$SRC_FILE copying missing libs to $$SRC_FILE"; \
+    echo "Analyzing $$SRC_FILE copying missing libs to $$TARGET_LIB_DIR"; \
     echo "Maybe outputting to $$MANIFEST_IN"; \
     \
     (mkdir $$TARGET_LIB_DIR || true); \
@@ -191,6 +191,7 @@ define copy_missing_libs
         fi; \
     done; \
     \
+    echo "Missing libs = $$missing_libs"; \
     for missing in $$missing_libs; do \
         find $(SO_SEARCH) -type f -name "$$missing" -exec cp {} $$TARGET_LIB_DIR \; ; \
         chmod +w $$TARGET_LIB_DIR/*.so ; \