remove test jars from the repository, generate them instead (#294)

* remove test jars from the repository, generate them instead

script now generates cpgs from sources, by compiling them and creating
jars, rather than having the jars checked into the repository

it also allows for 'pregenerated' jars like helloshiftleft, since we
don't want to invoke the full maven build

* regenerate cpgs

Author: mpollmeier

.gitattributes

@@ -1 +1 @@
-resources/cpgs/** filter=lfs diff=lfs merge=lfs -text
+resources/testcode/** filter=lfs diff=lfs merge=lfs -text

cpgqueryingtests/src/main/scala/io/shiftleft/cpgqueryingtests/codepropertygraph/CpgTestFixture.scala

@@ -7,7 +7,7 @@ import io.shiftleft.layers.{DataFlowRunner, EnhancementRunner}
 import io.shiftleft.semanticsloader.SemanticsLoader
 
 class CpgTestFixture(projectName: String) {
-  lazy val cpg = CpgLoader.load(s"resources/cpgs/$projectName/cpg.bin.zip",
+  lazy val cpg = CpgLoader.load(s"resources/testcode/cpgs/$projectName/cpg.bin.zip",
                                 CpgLoaderConfig().withOverflowConfig(OverflowDbConfig.disabled))
   new EnhancementRunner().run(cpg, new SerializedCpg())
   new DataFlowRunner(SemanticsLoader.emptySemantics).run(cpg, new SerializedCpg());

regenerate-test-cpgs.sh

@@ -1,18 +1,47 @@
 #!/usr/bin/env sh
 
-export CPG_HOME=`pwd`
+# stop on errors
+set -e
+set -o pipefail
+
+CPG_HOME=`pwd`
+TESTCODE_ROOT="$CPG_HOME/resources/testcode"
+SRC_ROOT="$TESTCODE_ROOT/sources"
+JARS_ROOT="$TESTCODE_ROOT/jars"
+PREGENERATED_JARS_ROOT="$TESTCODE_ROOT/jars-pregenerated"
+CPGS_ROOT="$TESTCODE_ROOT/cpgs"
 
 DEFAULT_JAVA2CPG_HOME=`readlink -f ../java2cpg`
 if [ -z "$JAVA2CPG_HOME" ]; then
   JAVA2CPG_HOME=$DEFAULT_JAVA2CPG_HOME
 fi
-
 echo "java2cpg home directory: $JAVA2CPG_HOME"
 
+# start fresh
+find "$SRC_ROOT" -name '*.class' -exec rm -rf {} \;
+rm -rf "$JARS_ROOT"
+cp -rp "$PREGENERATED_JARS_ROOT" "$JARS_ROOT"
+rm -rf "$CPGS_ROOT"
+mkdir -p "$CPGS_ROOT"
+
+# create jars 
+cd "$SRC_ROOT"
+for TESTCASE in `ls`; do
+  cd "$SRC_ROOT/$TESTCASE"
+  echo "creating jar for `pwd`"
+  javac -g *
+  jar -cvf "$JARS_ROOT/$TESTCASE.jar" .
+done;
+
 cd $JAVA2CPG_HOME
 sbt stage
 
-for jar in `find $CPG_HOME/resources/cpgs -name '*.jar'`; do
-  OUT_DIR=`dirname $jar`
-  ./java2cpg.sh $jar -o $OUT_DIR/cpg.bin.zip
+for jar in `find "$JARS_ROOT" -name '*.jar'`; do
+  TESTCASE=`basename "$jar" | sed s/\.jar//`
+  mkdir -p "$CPGS_ROOT/$TESTCASE"
+  ./java2cpg.sh "$jar" -o "$CPGS_ROOT/$TESTCASE/cpg.bin.zip"
 done;
+
+# cleanup
+find "$SRC_ROOT" -name '*.class' -exec rm -rf {} \;
+rm -rf "$JARS_ROOT"

resources/cpgs/expression/cpg.bin.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855b48ae931b2a1edec475b82a0c99aca8a2579db0ecbbf348d9bafb1b987f9b
+size 17827

resources/cpgs/expression/expression.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ff72c9b2237ce536c97c3d769bb09d8fbeab21d60146442e79a72a9728094d2
+size 17497

resources/cpgs/file/cpg.bin.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6da53f44619b8c959102faf3081cf51b15761d785671f01906f7fdae14572de2
+size 601600

resources/cpgs/file/file.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f343b8714136b564e623388996b581fca27a6a85a868fdfbdecbde5e54235ab
+size 19823

resources/cpgs/helloshiftleft/cpg.bin.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eba78b92b7dab475d85b2070dfc093354197af2b4ee01878c5bd5d65e79a91ae
+size 22581

resources/cpgs/memberaccesslinker/cpg.bin.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99f3feb90d21d42499ae85fe202bd5f6f06b8c445235d30b8ce243cd9801d024
+size 17808

resources/cpgs/memberaccesslinker/memberaccesslinker.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8a41b7438464e22e683caf41b819a5d9e0c6b45ff0db5ed1b528c1e655b87b
+size 16845

resources/cpgs/method/cpg.bin.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d00f710c4395d22c2bd4708041969bcf25d3f290573edf3a5a5f89ef218cd5a1
+size 26621

resources/cpgs/method/method.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4833e39f0a40642e976a30eab76b7a324b3fd91970c9dcf65b404fc031a62694
+size 22241

resources/cpgs/methodparameter/cpg.bin.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:637856006e87ba06ca594960e733e0d86bcc9d049704002d78476e1be030aa47
+size 32496494

resources/cpgs/methodparameter/methodparameter.jar

@@ -7,7 +7,7 @@ import io.shiftleft.layers.{DataFlowRunner, EnhancementRunner}
 import io.shiftleft.semanticsloader.SemanticsLoader
 
 class Fixture(projectName: String) {
-  val cpg = CpgLoader.load(s"resources/cpgs/$projectName/cpg.bin.zip",
+  val cpg = CpgLoader.load(s"resources/testcode/cpgs/$projectName/cpg.bin.zip",
                            CpgLoaderConfig().withOverflowConfig(OverflowDbConfig.disabled))
   new EnhancementRunner().run(cpg, new SerializedCpg())
   new DataFlowRunner(SemanticsLoader.emptySemantics)