refactor(interactive): Add more detail tracer for query handler #2431

Workflow file for this run

.github/workflows/hqps-db-ci.yml at eeb4932

	name: GraphScope GIE HighQPS DB CI

	on:
	# Trigger the workflow on push or pull request,
	# but only for the main branch
	workflow_dispatch:
	push:
	branches:
	- main
	paths:
	- 'flex/**'
	- 'interactive_engine/**'
	- '.github/workflows/hqps-db-ci.yml'
	pull_request:
	branches:
	- main
	paths:
	- 'flex/**'
	- 'interactive_engine/**'
	- '.github/workflows/hqps-db-ci.yml'

	concurrency:
	group: ${{ github.repository }}-${{ github.event.number \|\| github.head_ref \|\| github.sha }}-${{ github.workflow }}
	cancel-in-progress: true

	jobs:
	test-hqps-engine:
	runs-on: ubuntu-20.04
	if: ${{ github.repository == 'alibaba/GraphScope' }}
	container:
	image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
	steps:
	- uses: actions/checkout@v3

	- uses: actions/cache@v3
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-

	- uses: actions/cache@v3
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	~/.cache/sccache
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

	- name: Install latest libgrape-lite
	if: false
	run: \|
	git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
	cd /tmp/libgrape-lite
	mkdir -p build && cd build
	cmake ..
	make -j$(nproc)
	make install

	- name: Setup tmate session
	if: false
	uses: mxschmitt/action-tmate@v3

	- name: Build
	env:
	GIE_HOME: ${{ github.workspace }}/interactive_engine/
	HOME: /home/graphscope/
	run: \|
	cd ${GITHUB_WORKSPACE}/
	git submodule update --init
	cd ${GITHUB_WORKSPACE}/flex
	mkdir build && cd build
	cmake .. && sudo make -j$(nproc)
	sudo make install

	# cargo
	. /home/graphscope/.cargo/env
	which cargo

	# build compiler
	cd ${GIE_HOME}/
	mvn clean install -Pexperimental -DskipTests

	- name: Prepare dataset and workspace
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	run: \|
	# download dataset
	git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
	mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
	GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
	BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
	cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
	cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml
	mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
	cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
	cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml
	mkdir -p ${INTERACTIVE_WORKSPACE}/data/graph_algo
	cp ${GITHUB_WORKSPACE}/flex/tests/interactive/graph_algo_test.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
	cp ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml
	mkdir -p ${INTERACTIVE_WORKSPACE}/data/modern_graph
	cp ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml

	# load graph
	cd ${GITHUB_WORKSPACE}/flex/build
	export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
	GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/
	export FLEX_DATA_DIR=../interactive/examples/movies
	GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/movies/indices/
	export FLEX_DATA_DIR=../interactive/examples/graph_algo
	GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/graph_algo/indices/
	export FLEX_DATA_DIR=../interactive/examples/modern_graph
	GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml -l ../interactive/examples/modern_graph/bulk_load.yaml -d ${INTERACTIVE_WORKSPACE}/data/modern_graph/indices/

	- name: Test HQPS admin http service
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
	TMP_INTERACTIVE_WORKSPACE: /tmp/temp_workspace
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	rm -rf ${TMP_INTERACTIVE_WORKSPACE}
	cd ${GITHUB_WORKSPACE}/flex/build/
	SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
	BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
	mkdir -p ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/
	cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
	GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
	cd ${GITHUB_WORKSPACE}/flex/tests/hqps
	sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml
	bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml ${GS_TEST_DIR}
	sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml

	- name: Build and test Interactive Java/Python SDK
	env:
	FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
	TMP_INTERACTIVE_WORKSPACE: /tmp/temp_workspace
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/interactive/sdk/

	bash generate_sdk.sh -g java
	bash generate_sdk.sh -g python
	cd java
	mvn clean install -DskipTests
	cd ../python
	pip3 install -r test-requirements.txt
	pip3 install .

	rm -rf ${TMP_INTERACTIVE_WORKSPACE}
	cd ${GITHUB_WORKSPACE}/flex/build/
	SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
	BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
	mkdir -p ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/
	cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
	GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
	cd ${GITHUB_WORKSPACE}/flex/tests/hqps
	sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml
	bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml java
	bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml python
	sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml

	- name: Sample Query test
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/build
	export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
	./bin/bulk_loader -g ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
	-l ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml -d /tmp/csr-data-dir/
	rm -r /tmp/csr-data-dir/runtime/*
	rm -r /tmp/csr-data-dir/wal
	./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
	/tmp/csr-data-dir/

	- name: Run codegen test.
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	HOME : /home/graphscope/
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/bin

	for i in 1 2 3 4 5 6 7 8 9 10 11 12;
	do
	cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/"
	cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
	cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
	echo $cmd
	eval ${cmd} \|\| exit 1
	done

	for i in 1 2 3 4 5 6 7 8 9 11 12; # 10 is not supported now
	do
	cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/"
	cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
	cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
	echo $cmd
	eval ${cmd} \|\| exit 1
	done

	# test movie graph, 8,9,10 are not supported now
	# change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies
	sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml
	for i in 1 2 3 4 5 6 7 11 12 13 14 15;
	do
	cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/"
	cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
	cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
	echo $cmd
	eval ${cmd} \|\| exit 1
	done

	- name: Test cypher procedure generation and loading
	env:
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	PLUGIN_DIR: ${INTERACTIVE_WORKSPACE}/data/modern_graph/plugins
	FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	mkdir -p ${PLUGIN_DIR}

	cd ${GITHUB_WORKSPACE}/flex/bin
	./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \
	--ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml -o=${PLUGIN_DIR} \
	--procedure_name=get_person_name \
	--graph_schema_path=../interactive/examples/modern_graph/graph.yaml

	./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
	--ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml -o=${PLUGIN_DIR} \
	--procedure_name=count_vertex_num \
	--graph_schema_path=../interactive/examples/modern_graph/graph.yaml

	cd ${GITHUB_WORKSPACE}/flex/tests/interactive/
	bash test_plugin_loading.sh ./modern_graph_schema_v0_0.yaml \
	../../interactive/examples/modern_graph/bulk_load.yaml \
	${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml

	bash test_plugin_loading.sh ./modern_graph_schema_v0_1.yaml \
	../../interactive/examples/modern_graph/bulk_load.yaml \
	${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml

	- name: Run End-to-End cypher adhoc ldbc query test
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	HOME : /home/graphscope/
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
	export ENGINE_TYPE=hiactor
	# change the default_graph config in ./engine_config_test.yaml to ldbc
	sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
	bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
	${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml

	- name: Run End-to-End cypher adhoc movie query test
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	HOME : /home/graphscope/
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
	export ENGINE_TYPE=hiactor
	# change the default_graph config in ./engine_config_test.yaml to movies
	sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
	bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \
	${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml

	- name: Run End-to-End cypher adhoc graph_algo query test
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	HOME : /home/graphscope/
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
	export ENGINE_TYPE=hiactor
	# change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo
	sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml
	bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
	${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml

	- name: Run Gremlin test on modern graph
	env:
	GS_TEST_DIR: ${{ github.workspace }}/gstest
	INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
	LD_LIBRARY_PATH: /usr/local/lib
	run: \|
	cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
	export ENGINE_TYPE=hiactor
	sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./engine_config_test.yaml
	bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \
	${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml gremlin

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

refactor(interactive): Add more detail tracer for query handler #2431

Workflow file

refactor(interactive): Add more detail tracer for query handler #2431

Jobs

Run details

Workflow file for this run