Merge branch 'main' into zsy/rust62

Author: siyuan0322

.dockerignore

@@ -0,0 +1,84 @@
+# docker ignore the .git directory
+.git
+**/.git
+
+# Python
+__pycache__/
+*.pyc
+.ipynb_checkpoints
+*.egg-info/
+*.eggs
+python/*.eggs
+
+# coordinator
+coordinator/proto
+
+# python
+python/proto
+
+# Gar file
+*.gar
+
+# dot file
+*.dot
+
+# Protobuf GRPC
+*.pb.*
+*_pb2.py
+*_pb2_grpc.py
+
+# Visual Studio Code
+.vscode/
+.project
+.classpath
+.factorypath
+.settings
+
+# Mac OSX
+.DS_store
+
+# CLion
+.idea/
+cmake-build-*/
+**/*.iml
+**/target/
+
+# Workspace
+workspace/
+
+# generated_docs
+docs/source/reference/
+docs/reference/generated/
+docs/reference/networkx/generated/
+docs/_build/
+
+analytical_engine/build/
+
+# etcd data directory
+default.etcd/
+
+# outer build directory
+build/
+
+interactive_engine/rust-common/src/proto/*
+!interactive_engine/rust-common/src/proto/mod.rs
+**/generated/
+
+interactive_engine/executor/store/groot/src/db/proto/*
+!interactive_engine/executor/store/groot/src/db/proto/mod.rs
+interactive_engine/data_load_tools/dependency-reduced-pom.xml
+interactive_engine/gaia-adaptor/dependency-reduced-pom.xml
+interactive_engine/executor/Cargo.lock
+interactive_engine/executor/engine/pegasus/benchmark/src/graph/storage/clickhouse/pb_gen/*
+interactive_engine/executor/ir/Cargo.lock
+
+# java sdk
+java/**/dependency-reduced-pom.xml
+
+# GraphLearn
+coordinator/graphlearn*
+
+# Lock files
+Chart.lock
+**/Cargo.lock
+package.lock

.gitattributes

@@ -0,0 +1,3 @@
+analytical_engine/java/** linguist-vendored
+interactive_engine/groot-module/** linguist-vendored
+

.github/workflows/build-graphscope-wheels-linux.yml

@@ -39,12 +39,18 @@ jobs:
         if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi
         cat /etc/hosts
 
+        # change the version for nightly release
+        # 0.15.0 -> 0.15.0a20220808
+        time=$(date "+%Y%m%d")
+        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
+        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
+
         cd ${GITHUB_WORKSPACE}/k8s
         # build graphscope wheels
-        sudo -E -u graphscope make graphscope-py3-package
+        sudo -E -u runner make graphscope-py3-package
 
         # build client wheels
-        sudo -E -u graphscope make graphscope-client-py3-package
+        sudo -E -u runner make graphscope-client-py3-package
 
         # package
         cd ${GITHUB_WORKSPACE}
@@ -78,7 +84,6 @@ jobs:
         packages_dir: upload_pypi/
 
     - name: Publish distribution to PyPI
-      if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       uses: pypa/[email protected]
       with:
         user: __token__
@@ -117,6 +122,23 @@ jobs:
         # dataset image
         make dataset-image
 
+    - name: Release Nightly Image
+      if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
+        # graphscope image
+        sudo docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:nightly
+        sudo docker push ${{ env.GS_IMAGE }}:nightly
+        # jupyter image
+        sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.JUPYTER_IMAGE }}:nightly
+        sudo docker push ${{ env.JUPYTER_IMAGE }}:nightly
+        # dataset image
+        sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.DATASET_IMAGE }}:nightly
+        sudo docker push ${{ env.DATASET_IMAGE }}:nightly
+
     - name: Extract Tag Name
       if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       id: tag
@@ -129,19 +151,14 @@ jobs:
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        sudo docker tag graphscope/graphscope:${SHORT_SHA} \
-            ${{ env.GS_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        # graphscope image
+        sudo docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.GS_IMAGE }}:${{ steps.tag.outputs.TAG }}
-        sudo docker tag graphscope/graphscope:${SHORT_SHA} \
-            ${{ env.GS_IMAGE }}:latest
-        sudo docker push ${{ env.GS_IMAGE }}:latest
         # jupyter image
-        sudo docker tag graphscope/jupyter:${SHORT_SHA} \
-            ${{ env.JUPYTER_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.JUPYTER_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.JUPYTER_IMAGE }}:${{ steps.tag.outputs.TAG }}
         # dataset image
-        sudo docker tag graphscope/dataset:${SHORT_SHA} \
-            ${{ env.DATASET_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.DATASET_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.DATASET_IMAGE }}:${{ steps.tag.outputs.TAG }}
 
   ubuntu-python-test:

.github/workflows/build-graphscope-wheels-macos.yml

@@ -38,6 +38,12 @@ jobs:
         source ~/.graphscope_env
         echo ${CC}
 
+        # change the version for nightly release
+        # 0.15.0 -> 0.15.0a20220808
+        time=$(date "+%Y%m%d")
+        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
+        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
+
         # build graphscope server wheel
         cd ${GITHUB_WORKSPACE}/k8s
         sudo -E make graphscope-py3-package
@@ -83,6 +89,12 @@ jobs:
         python3 -c "import sys; print(sys.version)"
         source ~/.graphscope_env
 
+        # change the version for nightly release
+        # 0.15.0 -> 0.15.0a20220808
+        time=$(date "+%Y%m%d")
+        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
+        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
+
         # build graphscope client wheel
         cd ${GITHUB_WORKSPACE}/k8s
         make graphscope-client-py3-package
@@ -100,7 +112,7 @@ jobs:
 
   # Action gh-action-pypi-publish not support non-linux os.
   publish-wheels:
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
     runs-on: ubuntu-20.04
     needs: [build-wheels, build-client-wheels]
     strategy:

.github/workflows/ci.yml

@@ -358,6 +358,10 @@ jobs:
         # install pytest
         python3 -m pip install pytest pytest-cov
 
+    - name: Setup tmate session
+      uses: mxschmitt/action-tmate@v3
+      if: false
+
     - name: Run Python Test
       if: ${{ needs.changes.outputs.gae-python == 'true' }}
       env:

.github/workflows/gae.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.6.0
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.8.2
       options:
         --shm-size 4096m
     steps:
@@ -75,7 +75,7 @@ jobs:
         # default install to "/opt/graphscope"
         make gae ENABLE_JAVA_SDK=ON BUILD_TEST=ON
         # also make coordinator and client for python test
-        make coordinator && make client
+        make client && make coordinator
 
     - name: Run Cpp Test
       env:
@@ -91,7 +91,7 @@ jobs:
         mvn install -Dmaven.antrun.skip=true --quiet
 
         source ${GRAPHSCOPE_HOME}/conf/grape_jvm_opts
-        export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-0.1-shaded.jar
+        export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-0.16.0-shaded.jar
         cd ${GITHUB_WORKSPACE}/analytical_engine/build
         ../test/app_tests.sh --test_dir ${GS_TEST_DIR}
 
@@ -101,7 +101,7 @@ jobs:
         RUN_JAVA_TESTS: ON
         GRAPHSCOPE_HOME: /opt/graphscope
       run: |
-        export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-0.1-shaded.jar
+        export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-0.16.0-shaded.jar
         source ${GRAPHSCOPE_HOME}/conf/grape_jvm_opts
 
         cd ${GITHUB_WORKSPACE}/python

.github/workflows/gaia.yml

@@ -65,13 +65,13 @@ jobs:
     - name: Build Ir on Experimental Store
       run: |
         source ${HOME}/.bashrc
-        cd ${GITHUB_WORKSPACE}/interactive_engine/executor/ir/compiler
+        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler
         make build
 
     - name: Ir Unit Test
       run: |
         source ${HOME}/.bashrc
-        cd ${GITHUB_WORKSPACE}/interactive_engine/executor/ir/compiler && make test
+        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && make test
 
     - name: CommonType Unit Test
       run: |
@@ -90,7 +90,7 @@ jobs:
     - name: Ir Integration Test on Experimental Store
       run: |
         source ${HOME}/.bashrc
-        cd ${GITHUB_WORKSPACE}/interactive_engine/executor/ir/compiler && ./ir_exprimental_ci.sh
+        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_exprimental_ci.sh
 
     - name: Upload GIE log
       if: always()

.github/workflows/gss.yml

@@ -189,6 +189,7 @@ jobs:
         helm test ci --timeout 5m0s
         cd ${GITHUB_WORKSPACE}/python
         python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_after_restart
+
     - name: Clean
       if: always()
       run: |

.github/workflows/networkx-forward-algo-nightly.yml

@@ -16,7 +16,7 @@ jobs:
       run:
         shell: bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.6.0
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.8.2
       options:
         --shm-size 4096m
 

.github/workflows/nightly.yml

@@ -55,19 +55,3 @@ jobs:
       with:
         file: ./python/coverage.xml
         fail_ci_if_error: true
-
-  build-gss-image:
-    if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/[email protected]
-
-    - name: Add envs to GITHUB_ENV
-      run: |
-        short_sha=$(git rev-parse --short HEAD)
-        echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV
-
-    - name: Build GraphScope Store Image
-      run: |
-        cd ${GITHUB_WORKSPACE}
-        sudo make graphscope-store-image

.github/workflows/release.yml

@@ -1,6 +1,12 @@
 name: Cut Release
 
 on:
+  workflow_dispatch:
+  schedule:
+    # The notifications for scheduled workflows are sent to the user who
+    # last modified the cron syntax in the workflow file.
+    # Trigger the workflow at 03:00(CST) every day.
+    - cron: '00 19 * * *'
   push:
     tags:
       - 'v*'
@@ -14,7 +20,7 @@ env:
 
 jobs:
   release-gss-image:
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/[email protected]
@@ -29,22 +35,30 @@ jobs:
         cd ${GITHUB_WORKSPACE}
         sudo make graphscope-store-image
 
+    - name: Release Nightly Image
+      if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
+        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:nightly
+        sudo docker push ${{ env.GSS_IMAGE }}:nightly
+
     - name: Extract Tag Name
+      if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       id: tag
       run: echo ::set-output name=TAG::${GITHUB_REF#refs/tags/v}
 
     - name: Release Image
+      if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       env:
         docker_password: ${{ secrets.DOCKER_PASSWORD }}
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} \
-          ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} \
-          ${{ env.GSS_IMAGE }}:latest
-        sudo docker push ${{ env.GSS_IMAGE }}:latest
 
   release-helm-charts:
     if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}

Makefile

@@ -66,7 +66,7 @@ client: gle
 	pip3 install --user --editable $(WORKING_DIR)/python
 
 .PHONY: coordinator
-coordinator:
+coordinator: client
 	cd $(WORKING_DIR)/coordinator && \
 	pip3 install -r requirements.txt -r requirements-dev.txt --user && \
 	python3 setup.py build_builtin
@@ -98,14 +98,6 @@ ifneq ($(INSTALL_PREFIX), /usr/local)
 		sudo ln -sfn ${INSTALL_PREFIX}/lib/cmake/graphscope-analytical /usr/local/lib/cmake/graphscope-analytical; \
 	fi
 endif
-ifeq (${ENABLE_JAVA_SDK}, ON)
-	cd $(WORKING_DIR)/analytical_engine/java && \
-	mvn clean install -DskipTests --quiet && \
-	sudo cp ${WORKING_DIR}/analytical_engine/java/grape-runtime/target/native/libgrape-jni.* ${INSTALL_PREFIX}/lib/ && \
-	sudo cp ${WORKING_DIR}/analytical_engine/java/grape-runtime/target/grape-runtime-0.1-shaded.jar ${INSTALL_PREFIX}/lib/ && \
-	sudo mkdir -p ${INSTALL_PREFIX}/conf/ && \
-	sudo cp ${WORKING_DIR}/analytical_engine/java/grape_jvm_opts ${INSTALL_PREFIX}/conf/
-endif
 
 .PHONY: gie
 gie:

VERSION

@@ -1 +1 @@
-0.15.0
+0.16.0