Adds blame interaction graphs (#452)

This PR adds a graph abstraction for blame interaction data with different views on the blame data.
Using graphs allows us to analyze blame interactions from a graph-theoretic perspective.

Author: boehmseb

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
 
     - name: Install system dependencies for CI
       run: |
-         sudo apt-get install time git libgit2-dev ninja-build libyaml-dev
+         sudo apt-get install time git libgit2-dev ninja-build libyaml-dev libgraphviz-dev
 
     - name: Install dependencies
       run: |

.github/workflows/reviewdog.yml

@@ -18,6 +18,10 @@ jobs:
       with:
         python-version: 3.9
 
+    - name: Install system dependencies for CI
+      run: |
+         sudo apt-get install libgraphviz-dev
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_third_party = PyQt5,_pytest,argparse_utils,benchbuild,click,cryptography,distro,git,github,graphviz,jinja2,matplotlib,numpy,packaging,pandas,pkg_resources,plotly,plumbum,pydriller,pygit2,pygtrie,pylatex,pytest,requests,rich,scipy,seaborn,setuptools,sklearn,tabulate,yaml
+known_third_party = PyQt5,_pytest,argparse_utils,benchbuild,click,cryptography,distro,git,github,graphviz,jinja2,matplotlib,networkx,numpy,packaging,pandas,pkg_resources,plotly,plumbum,pydriller,pygit2,pygtrie,pylatex,pytest,requests,rich,scipy,seaborn,setuptools,sklearn,tabulate,yaml
 multi_line_output=3
 use_parentheses = True
 include_trailing_comma: True

.pylintrc

@@ -254,10 +254,14 @@ function-naming-style=snake_case
 #function-rgx=
 
 # Good variable names which should always be accepted, separated by a comma.
-good-names=i,
+good-names=a,
+           b,
+           i,
            j,
            k,
            n,
+           x,
+           y,
            ax,
            ex,
            Run,

mypy.ini

@@ -21,10 +21,10 @@ ignore_missing_imports = True
 [mypy-matplotlib.*]
 ignore_missing_imports = True
 
-[mypy-pandas.*]
+[mypy-networkx.*]
 ignore_missing_imports = True
 
-[mypy-numpy.*]
+[mypy-pandas.*]
 ignore_missing_imports = True
 
 [mypy-plotly.*]

requirements.txt

@@ -7,6 +7,7 @@ Jinja2>=3.0.1
 jupyter>=1.0.0
 kaleido>=0.2.1
 matplotlib>=3.1.2
+networkx>=2.5
 numpy>=1.21
 packaging>=20.1
 pandas>=0.22.0
@@ -17,6 +18,7 @@ pre-commit>=2.3.0
 PyDriller>=2.0
 pygit2>=0.28.2
 PyGithub>=1.47
+pygraphviz>=1.7
 pygtrie
 pylatex>=1.4.1
 PyQt5>=5.10.0,<5.14.0

tests/TEST_INPUTS/results/xz/BRE-BR-xz-xz-2f0bc9cd40_9e238675-ee7c-4325-8e9f-8ccf6fd3f05c_success.yaml

@@ -8,6 +8,7 @@
 from tests.test_utils import run_in_test_environment, UnitTestInputs
 from varats.data.reports.commit_report import CommitReport as CR
 from varats.paper_mgmt.paper_config import get_paper_config, load_paper_config
+from varats.projects.discover_projects import initialize_projects
 from varats.report.report import FileStatusExtension, ReportFilename
 from varats.utils.git_util import FullCommitHash, ShortCommitHash
 from varats.utils.settings import vara_cfg
@@ -16,6 +17,40 @@
 class TestCaseStudyRevisionLookupFunctions(unittest.TestCase):
     """Test if revision look up functions find the correct revisions."""
 
+    @classmethod
+    def setUpClass(cls):
+        initialize_projects()
+
+    @run_in_test_environment(
+        UnitTestInputs.PAPER_CONFIGS, UnitTestInputs.RESULT_FILES
+    )
+    def test_newest_processed_revision(self) -> None:
+        """Check whether the newest processed revision is correctly
+        identified."""
+        vara_cfg()['paper_config']['current_config'] = "test_revision_lookup"
+        load_paper_config()
+
+        newest_processed = MCS.newest_processed_revision_for_case_study(
+            get_paper_config().get_case_studies('brotli')[0], CR
+        )
+
+        self.assertEqual(
+            FullCommitHash('21ac39f7c8ca61c855be0bc38900abe7b5a0f67f'),
+            newest_processed
+        )
+
+    @run_in_test_environment(UnitTestInputs.PAPER_CONFIGS)
+    def test_newest_processed_revision_no_results(self) -> None:
+        """Check None is returned when no results are available."""
+        vara_cfg()['paper_config']['current_config'] = "test_revision_lookup"
+        load_paper_config()
+
+        newest_processed = MCS.newest_processed_revision_for_case_study(
+            get_paper_config().get_case_studies('brotli')[0], CR
+        )
+
+        self.assertIsNone(newest_processed)
+
     @run_in_test_environment(
         UnitTestInputs.PAPER_CONFIGS, UnitTestInputs.RESULT_FILES
     )

tests/TEST_INPUTS/results/xz/BRE-BR-xz-xz-c5c7ceb08a_77a6c5bc-e5c7-4532-8814-70dbcc6b5dda_success.yaml

@@ -0,0 +1,164 @@
+"""Test chord plot utils."""
+import typing as tp
+import unittest
+
+import numpy as np
+import numpy.typing as nptp
+
+from varats.plots.chord_plot_utils import (
+    _angular_to_cartesian,
+    _make_equilateral_triangle,
+    _make_bezier_curve,
+    _modulo_ab,
+    get_color_at,
+    _make_arc,
+    _calculate_ideogram_data,
+    _calculate_ribbon_data,
+    _calculate_arc_bounds,
+    _calculate_node_placements,
+)
+
+
+class TestChordPlotUtils(unittest.TestCase):
+    """Test chord plot utils."""
+
+    def _assert_point_equals(self, point_a, point_b):
+        self.assertAlmostEqual(point_a[0], point_b[0])
+        self.assertAlmostEqual(point_a[1], point_b[1])
+
+    def _assert_points_equal(self, points_a, points_b):
+        self.assertEqual(len(points_a), len(points_b))
+        for a, b in zip(points_a, points_b):
+            self._assert_point_equals(a, b)
+
+    def test_angular_to_cartesian(self):
+        result = _angular_to_cartesian([0, 0])
+        self._assert_point_equals([0, 0], result)
+
+        result = _angular_to_cartesian([1, 0])
+        self._assert_point_equals([1, 0], result)
+
+        result = _angular_to_cartesian([1, 0.5 * np.pi])
+        self._assert_point_equals([0, 1], result)
+
+        result = _angular_to_cartesian([0.5, np.pi])
+        self._assert_point_equals([-0.5, 0], result)
+
+    def test_equilateral_triangle(self):
+        result = _make_equilateral_triangle(np.array([0, 0]), np.array([1, 0]))
+        self._assert_point_equals([0.5, 0.5 * np.sqrt(3)], result)
+
+    def test_make_bezier_curve_parabola(self):
+        result = _make_bezier_curve(
+            np.array([[0, 0], [0.5, 0.5 * np.sqrt(3)], [1, 0]]), 5
+        )
+        # A quadratic Bézier curve is a parabola of the form a*t^2 + b*t + c.
+        # Since the first and last control point in this test have x=0,
+        # we can simplify this formula:
+        a = -np.sqrt(3)
+        b = np.sqrt(3)
+
+        def curve(t: float) -> float:
+            return a * t * t + b * t
+
+        self._assert_points_equal([(t, curve(t)) for t in np.linspace(0, 1, 5)],
+                                  result)
+
+    def test_make_arc(self):
+        result = _make_arc(np.array([1, 0]), np.array([-1, 0]), 5)
+        self._assert_points_equal([
+            (np.cos(t), np.sin(t)) for t in np.linspace(0, np.pi, 5)
+        ], result)
+
+    def test_modulo_ab(self):
+        self.assertAlmostEqual(3, _modulo_ab(3, 2, 4))
+        self.assertAlmostEqual(3, _modulo_ab(1, 2, 4))
+        self.assertAlmostEqual(3, _modulo_ab(5, 2, 4))
+        self.assertAlmostEqual(2, _modulo_ab(2, 2, 4))
+        self.assertAlmostEqual(2, _modulo_ab(4, 2, 4))
+
+    def test_get_color_at(self):
+        colorscale = [[0, "rgb(100.0, 0.0, 0.0)"], [1, "rgb(0.0, 0.0, 0.0)"]]
+        self.assertEqual("rgb(100.0, 0.0, 0.0)", get_color_at(colorscale, -0.5))
+        self.assertEqual("rgb(0.0, 0.0, 0.0)", get_color_at(colorscale, 1.5))
+        self.assertEqual("rgb(50.0, 0.0, 0.0)", get_color_at(colorscale, 0.5))
+        self.assertEqual("rgb(75.0, 0.0, 0.0)", get_color_at(colorscale, 0.25))
+
+    def test_calculate_ideogram_data(self):
+        ends, colors = _calculate_ideogram_data([1, 2, 3])
+        self._assert_points_equal([[0, np.pi / 3], [np.pi / 3, np.pi],
+                                   [np.pi, 2 * np.pi]], ends)
+        self.assertEqual(3, len(colors))
+
+    def test_calculate_ribbon_data(self):
+        node_a = "a"
+        node_b = "b"
+        node_c = "c"
+        node_data = {"color": 0, "info": ""}
+        edges = [
+            (node_a, node_c, {
+                "color": 0,
+                "info": "",
+                "size": 1
+            }),
+            (node_b, node_c, {
+                "color": 0,
+                "info": "",
+                "size": 2
+            }),
+        ]
+        nodes = [
+            (node_a, node_data),
+            (node_b, node_data),
+            (node_c, node_data),
+        ]
+        node_sizes = [1, 2, 3]
+        ideo_ends, ideo_colors = _calculate_ideogram_data([1, 2, 3])
+        bounds, colors = _calculate_ribbon_data(
+            nodes, edges, node_sizes, ideo_ends, ideo_colors
+        )
+        self._assert_points_equal([[0, np.pi / 3], [5 * np.pi / 3, 2 * np.pi]],
+                                  bounds[0])
+        self._assert_points_equal([[np.pi / 3, np.pi], [np.pi, 5 * np.pi / 3]],
+                                  bounds[1])
+        self.assertEqual(2, len(colors))
+
+    def test_calculate_node_placements(self):
+        placements = _calculate_node_placements([1, 2, 3])
+        self.assertEqual(0.5, placements[0])
+        self.assertEqual(2, placements[1])
+        self.assertEqual(4.5, placements[2])
+
+    def test_calculate_arc_bounds(self):
+        node_a = "a"
+        node_b = "b"
+        node_c = "c"
+        node_data = {"fill_color": 0, "line_color": 0, "info": "", "size": 0}
+        edges = [
+            (node_a, node_c, {
+                "color": 0,
+                "info": "",
+                "size": 1
+            }),
+            (node_b, node_a, {
+                "color": 0,
+                "info": "",
+                "size": 1
+            }),
+            (node_b, node_c, {
+                "color": 0,
+                "info": "",
+                "size": 2
+            }),
+        ]
+        nodes = [
+            (node_a, node_data),
+            (node_b, node_data),
+            (node_c, node_data),
+        ]
+        node_placements = _calculate_node_placements([1, 2, 3])
+
+        bounds = _calculate_arc_bounds(nodes, edges, node_placements)
+        self._assert_points_equal([[0.5, 0], [4.5, 0]], bounds[0])
+        self._assert_points_equal([[2, 0], [0.5, 0]], bounds[1])
+        self._assert_points_equal([[2, 0], [4.5, 0]], bounds[2])

tests/TEST_INPUTS/results/xz/BRE-BR-xz-xz-ef364d3abc_feeeecb2-1826-49e5-a188-d4d883f06d00_success.yaml

@@ -0,0 +1,107 @@
+"""Test blame interaction graphs."""
+
+import unittest
+
+import pytest
+
+from tests.test_utils import run_in_test_environment, UnitTestInputs
+from varats.data.reports.blame_interaction_graph import (
+    create_blame_interaction_graph,
+    create_file_based_interaction_graph,
+    get_author_data,
+)
+from varats.data.reports.blame_report import BlameReport
+from varats.paper_mgmt.case_study import (
+    newest_processed_revision_for_case_study,
+)
+from varats.paper_mgmt.paper_config import load_paper_config, get_paper_config
+from varats.projects.discover_projects import initialize_projects
+from varats.utils.settings import vara_cfg
+
+
+class TestBlameInteractionGraphs(unittest.TestCase):
+    """Test if blame interaction graphs are constructed correctly."""
+
+    @classmethod
+    def setUpClass(cls):
+        initialize_projects()
+
+    @run_in_test_environment(
+        UnitTestInputs.PAPER_CONFIGS, UnitTestInputs.RESULT_FILES
+    )
+    def test_blame_interaction_graph(self) -> None:
+        """Test whether blame interaction graphs are created correctly."""
+        vara_cfg()['paper_config']['current_config'] = "test_casestudy_status"
+        load_paper_config()
+
+        revision = newest_processed_revision_for_case_study(
+            get_paper_config().get_case_studies("xz")[0], BlameReport
+        )
+        blame_interaction_graph = create_blame_interaction_graph("xz", revision)
+
+        self.assertEqual(blame_interaction_graph.project_name, "xz")
+
+        cig = blame_interaction_graph.commit_interaction_graph()
+        self.assertEqual(124, len(cig.nodes))
+        self.assertEqual(928, len(cig.edges))
+
+        aig = blame_interaction_graph.author_interaction_graph()
+        self.assertEqual(1, len(aig.nodes))
+        self.assertEqual(0, len(aig.edges))
+
+        caig = blame_interaction_graph.commit_author_interaction_graph()
+        self.assertEqual(125, len(caig.nodes))
+        self.assertEqual(92, len(caig.edges))
+
+    @pytest.mark.slow
+    @run_in_test_environment(
+        UnitTestInputs.PAPER_CONFIGS, UnitTestInputs.RESULT_FILES
+    )
+    def test_file_based_interaction_graph(self) -> None:
+        """Test whether file-based interaction graphs are created correctly."""
+        vara_cfg()['paper_config']['current_config'] = "test_casestudy_status"
+        load_paper_config()
+
+        revision = newest_processed_revision_for_case_study(
+            get_paper_config().get_case_studies("xz")[0], BlameReport
+        )
+        blame_interaction_graph = create_file_based_interaction_graph(
+            "xz", revision
+        )
+
+        self.assertEqual(blame_interaction_graph.project_name, "xz")
+
+        cig = blame_interaction_graph.commit_interaction_graph()
+        self.assertEqual(482, len(cig.nodes))
+        self.assertEqual(16518, len(cig.edges))
+
+        aig = blame_interaction_graph.author_interaction_graph()
+        self.assertEqual(4, len(aig.nodes))
+        self.assertEqual(6, len(aig.edges))
+
+        caig = blame_interaction_graph.commit_author_interaction_graph()
+        self.assertEqual(486, len(caig.nodes))
+        self.assertEqual(509, len(caig.edges))
+
+    @run_in_test_environment(
+        UnitTestInputs.PAPER_CONFIGS, UnitTestInputs.RESULT_FILES
+    )
+    def test_get_author_data(self) -> None:
+        """Check whether author data is retrieved correctly from the author
+        interaction graph."""
+        vara_cfg()['paper_config']['current_config'] = "test_casestudy_status"
+        load_paper_config()
+
+        revision = newest_processed_revision_for_case_study(
+            get_paper_config().get_case_studies("xz")[0], BlameReport
+        )
+        blame_interaction_graph = create_blame_interaction_graph("xz", revision)
+
+        self.assertEqual(blame_interaction_graph.project_name, "xz")
+
+        aig = blame_interaction_graph.author_interaction_graph()
+        author_data = get_author_data(aig, "Lasse Collin")
+        self.assertEqual(author_data["node_attrs"]["author"], "Lasse Collin")
+        self.assertEqual(author_data["neighbors"], set())
+        self.assertEqual(0, len(author_data["in_attrs"]))
+        self.assertEqual(0, len(author_data["out_attrs"]))