Build wheel package on python3.10 and python3.11 (#2379)

.github/workflows/build-graphscope-wheels-linux.yml

@@ -160,7 +160,8 @@ jobs:
     needs: [build-wheels]
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        # The protobuf version of tenorflow and grpcio is conflicts in python 3.11
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/download-artifact@v3
@@ -175,6 +176,7 @@ jobs:
     - name: Prepare Environment
       shell: bash
       run: |
+        echo "Python version: ${{ matrix.python-version }}"
         # install graphscope-client
         cd artifacts
         tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
@@ -192,7 +194,11 @@ jobs:
         popd
 
         # install tensorflow
-        python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user
+        if [ "${{ matrix.python-version }}" == "3.10" ]; then
+          python3 -m pip install pytest "tensorflow<=2.8.4" "pandas" --user
+        else
+          python3 -m pip install pytest "tensorflow<=2.5.2" "pandas" --user
+        fi
         # install java
         sudo apt update -y && sudo apt install openjdk-11-jdk -y
 
@@ -248,7 +254,8 @@ jobs:
         popd
 
         # install tensorflow
-        python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user
+        python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0"  --user
+        python3 -m pip install -U wrapt --user
         # install jdk
         dnf install -y java-11-openjdk-devel
 

.github/workflows/build-graphscope-wheels-macos.yml

@@ -64,7 +64,7 @@ jobs:
     runs-on: macos-10.15
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3
@@ -117,7 +117,7 @@ jobs:
     needs: [build-wheels, build-client-wheels]
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/download-artifact@v3
@@ -164,7 +164,8 @@ jobs:
     needs: [build-wheels, build-client-wheels]
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9']
+        # The protobuf version of tenorflow and grpcio is conflicts in python 3.11
+        python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/download-artifact@v3
@@ -191,7 +192,11 @@ jobs:
         popd
 
         # install tensorflow
-        python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user
+        if [ "${{ matrix.python-version }}" == "3.10" ]; then
+          python3 -m pip install pytest "tensorflow<=2.8.4" "pandas" --user
+        else
+          python3 -m pip install pytest "tensorflow<=2.5.2" "pandas<1.5.0" --user
+        fi
 
     - name: Run Minimum Test
       env:

coordinator/requirements.txt

@@ -1,12 +1,15 @@
 etcd-distro>=3.5.1
 graphscope-client>=0.11.0
-grpcio<=1.43.0,>=1.40.0
-grpcio-tools<=1.43.0,>=1.40.0
+grpcio;python_version>="3.11"
+grpcio<=1.43.0,>=1.40.0;python_version<"3.11"
+grpcio-tools;python_version>="3.11"
+grpcio-tools<=1.43.0,>=1.40.0;python_version<"3.11"
 kubernetes~=12.0.1
-protobuf>=3.15.0,<=3.18.1
+protobuf;python_version>="3.11"
+protobuf>=3.15.0,<=3.18.1;python_version<"3.11"
 PyYAML
-vineyard>=0.11.6; sys_platform != "win32"
-vineyard-io>=0.11.6; sys_platform != "win32"
+vineyard>=0.11.1; sys_platform != "win32"
+vineyard-io>=0.11.1; sys_platform != "win32"
 prometheus-client>=0.14.1
 setuptools>=65.5.1
 packaging

coordinator/setup.py

@@ -373,8 +373,6 @@ def parse_version(root, **kwargs):
     },
     setup_requires=[
         "setuptools_scm>=5.0.0",
-        "grpcio<=1.43.0,>=1.40.0",
-        "grpcio-tools<=1.43.0,>=1.40.0",
     ],
     package_dir=parsed_package_dir(),
     packages=parsed_packages(),

k8s/internal/Makefile

@@ -127,6 +127,7 @@ graphscope-manylinux2014-py3:
 			cd /work/k8s/internal && \
 			make graphscope-manylinux2014-py3-nodocker'
 
+
 graphscope-client-manylinux2014-py3-nodocker:
 	cd $(WORKING_DIR)/../../learning_engine/graph-learn && \
 	git submodule update --init third_party/pybind11 && \
@@ -137,11 +138,17 @@ graphscope-client-manylinux2014-py3-nodocker:
 	cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \
 	make graphlearn_shared -j`nproc` && \
 	export LD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:$$LD_LIBRARY_PATH && \
-	for py in cp37-cp37m cp38-cp38 cp39-cp39 ; do \
+	for py in cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do \
 		cd $(WORKING_DIR)/../../python; \
 		export PATH=/opt/python/$$py/bin:$$PATH; \
 		pip3 install -U pip; \
-		pip3 install "numpy==1.18.5" "pandas<1.5.0" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" wheel "auditwheel==5.0.0"; \
+		if [ "$$py" == "cp311-cp311" ]; then \
+			pip3 install "numpy" "pandas" "grpcio" "grpcio-tools" wheel "auditwheel==5.0.0"; \
+		elif [ "$$py" == "cp310-cp310" ]; then \
+			 pip3 install "numpy==1.21.6" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" wheel "auditwheel==5.0.0"; \
+		else \
+			pip3 install "numpy==1.18.5" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" wheel "auditwheel==5.0.0"; \
+		fi; \
 		rm -rf build; \
 		rm -rf dist/*.whl; \
 		python3 setup.py bdist_wheel; \
@@ -160,8 +167,15 @@ graphscope-client-darwin-py3:
 		make graphlearn_shared -j`nproc` && \
 		export DYLD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:/usr/local/lib:$$DYLD_LIBRARY_PATH && \
 		cd $(WORKING_DIR)/../../python && \
+		py=$$(python3 -V 2>&1 | awk '{print $$2}' | awk -F '.' '{print $$1$$2}') && \
 		pip3 install -U pip && \
-		pip3 install "numpy==1.18.5" "pandas<1.5.0" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" delocate wheel && \
+		if [ "$$py" == "311" ]; then \
+			pip3 install "numpy" "pandas" "grpcio" "grpcio-tools" delocate wheel; \
+		elif [ "$$py" == "310" ]; then \
+			pip3 install "numpy==1.21.6" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" delocate wheel; \
+		else \
+			pip3 install "numpy==1.18.5" "pandas" "grpcio<=1.43.0,>=1.40.0" "grpcio-tools<=1.43.0,>=1.40.0" delocate wheel; \
+		fi; \
 		rm -rf build dist/*.whl || true && \
 		python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
 		for wheel in `ls dist/*.whl`; do \

k8s/utils/precompile.py

@@ -458,7 +458,7 @@ def parse_sys_args():
     )
     return parser.parse_args()
 
-WORKSPACE = os.path.join("/", tempfile.gettempprefix(), "gs", "builtin")
+WORKSPACE = Path(os.path.join("/", tempfile.gettempprefix(), "gs", "builtin")).resolve()
 
 if __name__ == "__main__":
     args = parse_sys_args()

python/graphscope/tests/minitest/test_min.py

@@ -126,6 +126,10 @@ def simple_flow(sess, ogbn_small_script):
     interactive = sess.gremlin(graph)
     papers = interactive.execute(ogbn_small_script).one()
 
+    # waiting for graphlearn support python3.10
+    if sys.version_info[1] >= 10:
+        return
+
     # GLE on ogbn_mag_small graph
     paper_features = []
     for i in range(128):
@@ -217,6 +221,10 @@ def test_demo_with_default_session(ogbn_small_script):
     sub_graph = sub_graph.add_column(ret1, {"kcore": "r"})
     sub_graph = sub_graph.add_column(ret2, {"tc": "r"})
 
+    # waiting for graphlearn support python3.10
+    if sys.version_info[1] >= 10:
+        return
+
     # GLE on ogbn_mag_small graph
     paper_features = []
     for i in range(128):

python/requirements.txt

@@ -1,7 +1,9 @@
 Cython==3.0a6
 gremlinpython>=3.4.9
-grpcio<=1.43.0,>=1.40.0
-grpcio-tools<=1.43.0,>=1.40.0
+grpcio;python_version>="3.11"
+grpcio<=1.43.0,>=1.40.0;python_version<"3.11"
+grpcio-tools;python_version>="3.11"
+grpcio-tools<=1.43.0,>=1.40.0;python_version<"3.11"
 kubernetes
 msgpack
 nest_asyncio
@@ -11,12 +13,14 @@ numpy
 orjson
 packaging
 pandas
-protobuf>=3.15.0,<=3.18.1
+protobuf;python_version>="3.11"
+protobuf>=3.15.0,<=3.18.1;python_version<"3.11"
 psutil
-pyarrow<=6.0.0
+pyarrow;python_version>="3.11"
+pyarrow<=6.0.0;python_version<"3.11"
 pysimdjson
 PyYAML
 scipy
 tqdm
-vineyard>=0.5.2; sys_platform != "win32"
-vineyard-io>=0.5.2; sys_platform != "win32"
+vineyard>=0.11.1; sys_platform != "win32"
+vineyard-io>=0.11.1; sys_platform != "win32"

python/setup.py

@@ -288,9 +288,6 @@ def parse_version(root, **kwargs):
     },
     setup_requires=[
         "setuptools_scm>=5.0.0",
-        "grpcio<=1.43.0,>=1.40.0",
-        "grpcio-tools<=1.43.0,>=1.40.0",
-        "numpy",
     ],
     package_dir=resolve_graphscope_package_dir(),
     packages=find_graphscope_packages(),

scripts/install_deps.sh

@@ -398,6 +398,7 @@ write_envs_config() {
       echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl"
       echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib"
       echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib"
+      echo "export LDFLAGS=\"$LDFLAGS -Wl,-rpath,${homebrew_prefix}/opt/libomp/lib -L${homebrew_prefix}/opt/libomp/lib\""
     } >> ${OUTPUT_ENV_FILE}
 
   elif [[ "${PLATFORM}" == *"Ubuntu"* ]]; then
@@ -556,7 +557,7 @@ install_cppkafka() {
 
   if [[ "${PLATFORM}" == *"Darwin"* ]]; then
     declare -r homebrew_prefix=$(brew --prefix)
-    export LDFLAGS="-L${homebrew_prefix}/opt/openssl@3/lib"
+    export LDFLAGS="${LDFLAGS} -L${homebrew_prefix}/opt/openssl@3/lib"
     export CPPFLAGS="-I${homebrew_prefix}/opt/openssl@3/include"
   fi
 
@@ -762,6 +763,7 @@ install_dependencies() {
     export CXX=${homebrew_prefix}/opt/llvm/bin/clang++
     export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}
     export CPPFLAGS=-I${homebrew_prefix}/opt/llvm/include
+    export LDFLAGS="${LDFLAGS} -Wl,-rpath,${homebrew_prefix}/opt/libomp/lib -L${homebrew_prefix}/opt/libomp/lib"
   fi
 
   log "Installing python packages for vineyard codegen."