More spec cleanup and json generation (#1204)

* Delete script that is no longer needed

* Boilerplate for `schema2json`

* demonstrate access to schema

* json4s example code

* Output node properties, taking into account inheritance

* Remove PROPAGATE edge and ALIAS property

* Remove `DetachedTrackingPoint`

* Remove ARRAY_INITIALIZER

* format

* Move `MetaData` into its own class

* Make version field accessible

Author: fabsx00

build.sbt

@@ -75,6 +75,7 @@ lazy val console = Projects.console
 lazy val fuzzyc2cpg = Projects.fuzzyc2cpg
 lazy val fuzzyc2cpgtests = Projects.fuzzyc2cpgtests
 lazy val macros = Projects.macros
+lazy val schema2json = Projects.schema2json
 
 // Once sbt-scalafmt is at version > 2.x, use scalafmtAll
 addCommandAlias("format", ";scalafixAll OrganizeImports;scalafmt;test:scalafmt")

codepropertygraph/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/DiffGraphTest.scala

@@ -2,8 +2,8 @@ package io.shiftleft.codepropertygraph.cpgloading
 
 import io.shiftleft.OverflowDbTestInstance
 import io.shiftleft.codepropertygraph.generated._
-import io.shiftleft.codepropertygraph.generated.edges.Propagate
-import io.shiftleft.codepropertygraph.generated.nodes.{MethodParameterIn, MethodParameterOut, NewNode, StoredNode}
+import io.shiftleft.codepropertygraph.generated.edges.ReachingDef
+import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, MethodParameterIn, NewNode, StoredNode}
 import io.shiftleft.passes.{DiffGraph, IntervalKeyPool}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
@@ -18,8 +18,8 @@ class DiffGraphTest extends AnyWordSpec with Matchers {
       // setup existing graph
       // add x and y nodes to graph
       val x = graph + (MethodParameterIn.Label, MethodParameterIn.Properties.Code -> "x")
-      val y = graph + (MethodParameterOut.Label, MethodParameterOut.Properties.Code -> "old y code")
-      val x2y = x --- (Propagate.Label, Propagate.Properties.Alias -> true) --> y
+      val y = graph + (Identifier.Label, Identifier.Properties.Code -> "old y code")
+      val x2y = x --- (ReachingDef.Label, ReachingDef.Properties.Variable -> "true") --> y
 
       // make diffgraph
       val diffBuilder = DiffGraph.newBuilder
@@ -39,15 +39,15 @@ class DiffGraphTest extends AnyWordSpec with Matchers {
       diffBuilder.addNodeProperty(y.asInstanceOf[StoredNode], PropertyNames.ORDER, Int.box(123))
       diffBuilder.addNodeProperty(y.asInstanceOf[StoredNode], PropertyNames.CODE, "new y code")
 
-      diffBuilder.addEdgeProperty(x2y, PropertyNames.ALIAS, JBoolean.FALSE)
+      diffBuilder.addEdgeProperty(x2y, PropertyNames.VARIABLE, JBoolean.FALSE)
       val diffGraph = diffBuilder.build()
       // apply diffgraph with undoable = true
       val appliedDiffGraph = DiffGraph.Applier.applyDiff(diffGraph, graph, true, None)
       val inverseDiffGraph = appliedDiffGraph.inverseDiffGraph.get
       val changes = inverseDiffGraph.iterator.toList
       import DiffGraph.Change._
       val List(
-        SetEdgeProperty(_, PropertyNames.ALIAS, JBoolean.TRUE), // restore old edge property value
+        SetEdgeProperty(_, PropertyNames.VARIABLE, "true"), // restore old edge property value
         SetNodeProperty(_, PropertyNames.CODE, "old y code"), // restore old Y property value
         RemoveNodeProperty(_, PropertyNames.ORDER), // remove newly added property
         RemoveEdge(_), // remove x -> a

codepropertygraph/src/test/scala/io/shiftleft/passes/CpgOverlayIntegrationTest.scala

@@ -3,8 +3,8 @@ package io.shiftleft.passes
 import io.shiftleft.OverflowDbTestInstance
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated._
-import io.shiftleft.codepropertygraph.generated.edges.Propagate
-import io.shiftleft.codepropertygraph.generated.nodes.{MethodParameterIn, MethodParameterOut, StoredNode}
+import io.shiftleft.codepropertygraph.generated.edges.ReachingDef
+import io.shiftleft.codepropertygraph.generated.nodes.{Identifier, MethodParameterIn, StoredNode}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import overflowdb._
@@ -45,13 +45,13 @@ class CpgOverlayIntegrationTest extends AnyWordSpec with Matchers {
       // 1) add a new node
       val addNodeInverse = applyDiffAndGetInverse(cpg)(
         _.addNode(
-          nodes.NewMethodParameterOut().code(null)
+          nodes.NewIdentifier().code(null)
         ))
       cpg.graph.nodeCount shouldBe 2
       val additionalNode = cpg.graph.V
-        .label(MethodParameterOut.Label)
+        .label(Identifier.Label)
         .collectFirst {
-          case mpe: MethodParameterOut if mpe.code == null => mpe
+          case mpe: nodes.Identifier if mpe.code == null => mpe
         }
         .get
         .asInstanceOf[StoredNode]
@@ -61,15 +61,15 @@ class CpgOverlayIntegrationTest extends AnyWordSpec with Matchers {
         _.addEdge(
           src = initialNode,
           dst = additionalNode,
-          edgeLabel = Propagate.Label,
-          properties = Seq(Propagate.PropertyNames.Alias -> Boolean.box(true))
+          edgeLabel = ReachingDef.Label,
+          properties = Seq(ReachingDef.PropertyNames.Variable -> "true")
         ))
       val addEdge2Inverse = applyDiffAndGetInverse(cpg)(
         _.addEdge(
           src = initialNode,
           dst = additionalNode,
-          edgeLabel = Propagate.Label,
-          properties = Seq(Propagate.PropertyNames.Alias -> Boolean.box(false))
+          edgeLabel = ReachingDef.Label,
+          properties = Seq(ReachingDef.PropertyNames.Variable -> "false")
         ))
       def initialNodeOutEdges = initialNode.outE.toList
       initialNodeOutEdges.size shouldBe 2
@@ -95,7 +95,7 @@ class CpgOverlayIntegrationTest extends AnyWordSpec with Matchers {
       // 2) remove edges - they don't have ids and are therefor disambiguated by their property hash
       DiffGraph.Applier.applyDiff(addEdge2Inverse, cpg)
       initialNodeOutEdges.size shouldBe 1
-      initialNode.outE.property(Properties.ALIAS).toList shouldBe List(true)
+      initialNode.outE.property(Properties.VARIABLE).toList shouldBe List("true")
       DiffGraph.Applier.applyDiff(addEdge1Inverse, cpg)
       initialNodeOutEdges.size shouldBe 0
 

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/language/nodemethods/TrackingPointMethods.scala

@@ -22,9 +22,8 @@ class TrackingPointMethods[NodeType <: nodes.TrackingPoint](val node: NodeType)
     * */
   def astNode: nodes.AstNode =
     node match {
-      case n: nodes.AstNode               => n
-      case n: nodes.DetachedTrackingPoint => n.cfgNode
-      case _                              => ??? //TODO markus/fabs?
+      case n: nodes.AstNode => n
+      case _                => ??? //TODO markus/fabs?
     }
 
   def reachableBy[NodeType <: nodes.TrackingPoint](sourceTravs: Traversal[NodeType]*)(

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/language/package.scala

@@ -22,12 +22,9 @@ package object language {
   implicit def trackingPointToAstNodeMethods(node: nodes.TrackingPoint) =
     new AstNodeMethods(trackingPointToAstNode(node))
 
-  implicit def trackingPointToAstNode(node: nodes.TrackingPoint): nodes.AstNode =
-    node match {
-      case n: nodes.AstNode               => n
-      case n: nodes.DetachedTrackingPoint => n.cfgNode
-      case _                              => ??? //TODO markus/fabs?
-    }
+  implicit def trackingPointToAstNode(node: nodes.TrackingPoint): nodes.AstNode = {
+    node.astNode
+  }
 
   implicit def toDdgNodeDot[A](a: A)(implicit f: A => Traversal[nodes.Method]): DdgNodeDot =
     new DdgNodeDot(f(a))

project/Projects.scala

@@ -13,4 +13,5 @@ object Projects {
   lazy val fuzzyc2cpg = project.in(file("fuzzyc2cpg"))
   lazy val fuzzyc2cpgtests = project.in(file("fuzzyc2cpg-tests"))
   lazy val macros = project.in(file("macros"))
+  lazy val schema2json = project.in(file("schema2json"))
 }

regenerate-test-cpgs.sh

@@ -20,8 +20,8 @@ object Base {
         valueType = ValueTypes.STRING,
         cardinality = Cardinality.One,
         comment = """A version, given as a string. Used, for example, in the META_DATA node to
-            |indicate which version of the CPG spec this CPG conforms to
-            |""".stripMargin
+                    |indicate which version of the CPG spec this CPG conforms to
+                    |""".stripMargin
       )
       .protoId(13)
 
@@ -310,33 +310,6 @@ object Base {
       .addProperties(name, signature)
       .extendz(cfgNode)
 
-    val language = builder
-      .addProperty(
-        name = "LANGUAGE",
-        valueType = ValueTypes.STRING,
-        cardinality = Cardinality.One,
-        comment = "the CPG language frontend that generated this CPG"
-      )
-      .protoId(19)
-
-    val overlays = builder
-      .addProperty(
-        name = "OVERLAYS",
-        valueType = ValueTypes.STRING,
-        cardinality = Cardinality.List,
-        comment = "Names of overlays applied to this graph, in order of application"
-      )
-      .protoId(118)
-
-    val metaData: NodeType = builder
-      .addNodeType(
-        name = "META_DATA",
-        comment = """Node to save meta data about the graph on its properties.
-            |Exactly one node of this type per graph""".stripMargin
-      )
-      .protoId(39)
-      .addProperties(language, version, overlays, hash)
-
     val file: NodeType = builder
       .addNodeType(
         name = "FILE",
@@ -578,14 +551,6 @@ object Base {
       .addProperties(name, signature, fullName, methodFullName)
       .extendz(astNode)
 
-    val arrayInitializer: NodeType = builder
-      .addNodeType(
-        name = "ARRAY_INITIALIZER",
-        comment = "Initialization construct for arrays"
-      )
-      .protoId(14)
-      .extendz(astNode)
-
     val methodRef: NodeType = builder
       .addNodeType(
         name = "METHOD_REF",
@@ -885,7 +850,6 @@ object Base {
     controlStructure
       .addOutEdge(edge = ast, inNode = literal, cardinalityIn = Cardinality.One)
       .addOutEdge(edge = ast, inNode = modifier)
-      .addOutEdge(edge = ast, inNode = arrayInitializer)
       .addOutEdge(edge = ast, inNode = callNode, cardinalityIn = Cardinality.One)
       .addOutEdge(edge = ast, inNode = local)
       .addOutEdge(edge = ast, inNode = identifier, cardinalityIn = Cardinality.ZeroOrOne)
@@ -907,7 +871,6 @@ object Base {
       .addOutEdge(edge = condition, inNode = jumpTarget)
       .addOutEdge(edge = condition, inNode = unknown)
       .addOutEdge(edge = condition, inNode = controlStructure)
-      .addOutEdge(edge = condition, inNode = arrayInitializer)
       .addOutEdge(edge = cfg, inNode = callNode)
       .addOutEdge(edge = cfg, inNode = identifier)
       .addOutEdge(edge = cfg, inNode = fieldIdentifier)
@@ -948,7 +911,6 @@ object Base {
       .addOutEdge(edge = ast, inNode = literal)
       .addOutEdge(edge = ast, inNode = member)
       .addOutEdge(edge = ast, inNode = modifier)
-      .addOutEdge(edge = ast, inNode = arrayInitializer)
       .addOutEdge(edge = ast, inNode = callNode)
       .addOutEdge(edge = ast, inNode = local)
       .addOutEdge(edge = ast, inNode = identifier)

schema/src/main/scala/io/shiftleft/codepropertygraph/schema/Base.scala

@@ -9,6 +9,7 @@ import overflowdb.schema.{Schema, SchemaBuilder}
 class CpgSchema(builder: SchemaBuilder) {
   // the foundation
   val base = Base(builder)
+  val metaData = MetaData(builder, base)
   val enhancements = Enhancements(builder, base)
 
   // everything else

schema/src/main/scala/io/shiftleft/codepropertygraph/schema/CpgSchema.scala

@@ -70,15 +70,6 @@ object Enhancements {
       )
       .protoId(57)
 
-    val alias = builder
-      .addProperty(
-        name = "ALIAS",
-        valueType = ValueTypes.BOOLEAN,
-        cardinality = Cardinality.One,
-        comment = "Defines whether a PROPAGATE edge creates an alias"
-      )
-      .protoId(1)
-
     val variable = builder
       .addProperty(
         name = "VARIABLE",
@@ -153,14 +144,6 @@ object Enhancements {
       )
       .protoId(28)
 
-    val propagate = builder
-      .addEdgeType(
-        name = "PROPAGATE",
-        comment = "Encodes propagation of data from on node to another. The ALIAS property is deprecated."
-      )
-      .protoId(1)
-      .addProperties(alias)
-
     val reachingDef = builder
       .addEdgeType(
         name = "REACHING_DEF",
@@ -192,18 +175,6 @@ object Enhancements {
 
 // node types
 
-    val detachedTrackingPoint: NodeType = builder
-      .addNodeType(
-        name = "DETACHED_TRACKING_POINT",
-        comment = ""
-      )
-      .protoId(1001)
-      .addProperties()
-      .extendz(trackingPoint)
-
-    detachedTrackingPoint
-      .addContainedNode(cfgNode, "cfgNode", Cardinality.One)
-
     val binding: NodeType = builder
       .addNodeType(
         name = "BINDING",
@@ -779,8 +750,6 @@ object Enhancements {
       .addOutEdge(edge = reachingDef, inNode = methodRef)
 
     methodParameterIn
-      .addOutEdge(edge = propagate, inNode = methodParameterOut)
-      .addOutEdge(edge = propagate, inNode = methodReturn)
       .addOutEdge(edge = evalType, inNode = tpe, cardinalityOut = Cardinality.One)
       .addOutEdge(edge = reachingDef, inNode = callNode)
       .addOutEdge(edge = reachingDef, inNode = ret)

schema/src/main/scala/io/shiftleft/codepropertygraph/schema/Enhancements.scala

@@ -0,0 +1,42 @@
+package io.shiftleft.codepropertygraph.schema
+
+import overflowdb.schema.{Cardinality, NodeType, SchemaBuilder}
+import overflowdb.storage.ValueTypes
+
+object MetaData {
+
+  def apply(builder: SchemaBuilder, base: Base.Schema) = new Schema(builder, base)
+
+  class Schema(builder: SchemaBuilder, base: Base.Schema) {
+    import base._
+
+    val overlays = builder
+      .addProperty(
+        name = "OVERLAYS",
+        valueType = ValueTypes.STRING,
+        cardinality = Cardinality.List,
+        comment = "Names of overlays applied to this graph, in order of application"
+      )
+      .protoId(118)
+
+    val language = builder
+      .addProperty(
+        name = "LANGUAGE",
+        valueType = ValueTypes.STRING,
+        cardinality = Cardinality.One,
+        comment = "the CPG language frontend that generated this CPG"
+      )
+      .protoId(19)
+
+    val metaData: NodeType = builder
+      .addNodeType(
+        name = "META_DATA",
+        comment = """Node to save meta data about the graph on its properties.
+                    |Exactly one node of this type per graph""".stripMargin
+      )
+      .protoId(39)
+      .addProperties(language, version, overlays, hash)
+
+  }
+
+}

schema/src/main/scala/io/shiftleft/codepropertygraph/schema/MetaData.scala

@@ -0,0 +1 @@
+./schema2json/target/universal/stage/bin/schema2json