Feat: drop origin from issue listing

backend/kernelCI_app/queries/issues.py

@@ -4,8 +4,11 @@
 from kernelCI_app.helpers.database import dict_fetchall
 from kernelCI_app.helpers.logger import log_message
 import typing_extensions
+from kernelCI_app.models import Issues
 from kernelCI_app.helpers.build import valid_do_not_exist_exception, valid_status_field
 from kernelCI_app.constants.general import SCHEMA_VERSION_ENV
+from datetime import datetime
+from django.db.models import Q
 
 
 @typing_extensions.deprecated(
@@ -84,3 +87,36 @@ def get_issue_tests(*, issue_id: str, version: int) -> list[dict]:
     with connection.cursor() as cursor:
         cursor.execute(query, params)
         return dict_fetchall(cursor)
+
+
+def get_issue_listing_data(
+    *,
+    interval_date: datetime,
+    culprit_code: bool | None,
+    culprit_harness: bool | None,
+    culprit_tool: bool | None,
+) -> list[dict]:
+    filters = Q(field_timestamp__gte=interval_date)
+
+    culprit_query = Q()
+
+    if culprit_code:
+        culprit_query |= Q(culprit_code=True)
+    if culprit_harness:
+        culprit_query |= Q(culprit_harness=True)
+    if culprit_tool:
+        culprit_query |= Q(culprit_tool=True)
+
+    filters = filters & culprit_query
+
+    issues_records = Issues.objects.values(
+        "id",
+        "field_timestamp",
+        "comment",
+        "version",
+        "origin",
+        "culprit_code",
+        "culprit_harness",
+        "culprit_tool",
+    ).filter(filters)
+    return issues_records

backend/kernelCI_app/typeModels/issueListing.py

@@ -1,4 +1,5 @@
 from pydantic import BaseModel
+from kernelCI_app.typeModels.commonListing import ListingQueryParameters
 
 from kernelCI_app.typeModels.databases import (
     Issue__Comment,
@@ -8,14 +9,22 @@
     Issue__Id,
     Issue__Version,
     Timestamp,
+    Origin,
 )
 from kernelCI_app.typeModels.issues import FirstIncident
 
 
+class IssueListingQueryParameters(ListingQueryParameters):
+    culprit_code: bool | None = False
+    culprit_harness: bool | None = False
+    culprit_tool: bool | None = False
+
+
 class IssueListingItem(BaseModel):
     field_timestamp: Timestamp
     id: Issue__Id
     comment: Issue__Comment
+    origin: Origin
     version: Issue__Version
     culprit_code: Issue__CulpritCode
     culprit_tool: Issue__CulpritTool

backend/kernelCI_app/unitTests/issues_test.py

@@ -17,18 +17,52 @@
 
 client = IssueClient()
 
+CULPRIT_CODE = {
+    "culpritCode": True,
+    "excludes_fields": ["culprit_tool", "culprit_harness"],
+}
+
+CULPRIT_TOOL = {
+    "culpritTool": True,
+    "excludes_fields": ["culprit_code", "culprit_harness"],
+}
+
+CULPRIT_HARNESS = {
+    "culpritHarness": True,
+    "excludes_fields": ["culprit_code", "culprit_tool"],
+}
+
+CULPRIT_CODE_AND_TOOL = {
+    "culpritCode": True,
+    "culpritTool": True,
+    "excludes_fields": ["culprit_harness"],
+}
+
+
+def pytest_generate_tests(metafunc):
+    issues_listing_base_cases = [
+        (5, CULPRIT_CODE, HTTPStatus.OK, False),
+        (5, CULPRIT_TOOL, HTTPStatus.OK, False),
+        (5, CULPRIT_HARNESS, HTTPStatus.OK, False),
+        (-5, None, HTTPStatus.BAD_REQUEST, True),
+    ]
+
+    if metafunc.config.getoption("--run-all"):
+        issues_listing_base_cases += [
+            (5, CULPRIT_CODE_AND_TOOL, HTTPStatus.OK, False),
+            (5, None, HTTPStatus.OK, False),
+        ]
+
+    if "issue_listing_input" in metafunc.fixturenames:
+        metafunc.parametrize("issue_listing_input", issues_listing_base_cases)
+
 
 @online
-@pytest.mark.parametrize(
-    "origin, interval_in_day, status_code, has_error_body",
-    [
-        ("maestro", 7, HTTPStatus.OK, False),
-        ("maestro", -5, HTTPStatus.BAD_REQUEST, True),
-        ("invalid origin", 5, HTTPStatus.OK, True),
-    ],
-)
-def test_list(origin, interval_in_day, status_code, has_error_body):
-    response = client.get_issues_list(origin=origin, interval_in_days=interval_in_day)
+def test_list(pytestconfig, issue_listing_input):
+    interval_in_day, culprit_data, status_code, has_error_body = issue_listing_input
+    response = client.get_issues_list(
+        interval_in_days=interval_in_day, culprit_data=culprit_data
+    )
     content = string_to_json(response.content.decode())
     assert_status_code_and_error_response(
         response=response,
@@ -46,6 +80,15 @@ def test_list(origin, interval_in_day, status_code, has_error_body):
             fields=issues_expected_fields, response_content=content["issues"][0]
         )
 
+        if culprit_data is not None:
+            for culprit in culprit_data["excludes_fields"]:
+                assert not content["issues"][0][culprit]
+
+            if pytestconfig.getoption("--run-all") and len(content["issues"]) > 1:
+                for culprit in culprit_data["excludes_fields"]:
+                    for issue in content["issues"][1:]:
+                        assert not issue[culprit]
+
 
 @online
 @pytest.mark.parametrize(

backend/kernelCI_app/unitTests/utils/client/issueClient.py

@@ -6,10 +6,12 @@
 
 class IssueClient(BaseClient):
     def get_issues_list(
-        self, *, origin: str | None, interval_in_days: int | None
+        self, *, interval_in_days: int | None, culprit_data: dict | None
     ) -> requests.Response:
         path = reverse("issue")
-        query = {"origin": origin, "intervalInDays": interval_in_days}
+        if culprit_data is None:
+            culprit_data = {}
+        query = {"intervalInDays": interval_in_days, **culprit_data}
         url = self.get_endpoint(path=path, query=query)
         return requests.get(url)
 

backend/kernelCI_app/views/issueView.py

@@ -9,30 +9,14 @@
     create_api_error_response,
 )
 from kernelCI_app.helpers.issueExtras import assign_issue_first_seen
-from kernelCI_app.models import Issues
-from kernelCI_app.typeModels.commonListing import ListingQueryParameters
+from kernelCI_app.queries.issues import get_issue_listing_data
 from kernelCI_app.typeModels.issueListing import (
     IssueListingResponse,
+    IssueListingQueryParameters,
 )
 from kernelCI_app.typeModels.issues import FirstIncident, ProcessedExtraDetailedIssues
 
 
-def get_issue_listing_data(origin: str, interval_date):
-    issues_records = Issues.objects.values(
-        "id",
-        "field_timestamp",
-        "comment",
-        "version",
-        "culprit_code",
-        "culprit_harness",
-        "culprit_tool",
-    ).filter(
-        origin=origin,
-        field_timestamp__gte=interval_date,
-    )
-    return issues_records
-
-
 class IssueView(APIView):
     def __init__(self):
         self.issue_records: list[dict[str]] = []
@@ -47,29 +31,36 @@ def _format_processing_for_response(self) -> None:
             self.first_incidents[issue_extras_id] = issue_extras_data["first_incident"]
 
     @extend_schema(
-        parameters=[ListingQueryParameters],
+        parameters=[IssueListingQueryParameters],
         responses=IssueListingResponse,
         methods=["GET"],
     )
     def get(self, _request) -> Response:
         try:
-            request_params = ListingQueryParameters(
-                origin=(_request.GET.get("origin")),
+            request_params = IssueListingQueryParameters(
                 interval_in_days=_request.GET.get("intervalInDays"),
+                culprit_code=_request.GET.get("culpritCode"),
+                culprit_harness=_request.GET.get("culpritHarness"),
+                culprit_tool=_request.GET.get("culpritTool"),
             )
         except ValidationError as e:
             return Response(data=e.json(), status=HTTPStatus.BAD_REQUEST)
 
-        origin_param = request_params.origin
         interval_in_days = request_params.interval_in_days
+        culprit_code = request_params.culprit_code
+        culprit_harness = request_params.culprit_harness
+        culprit_tool = request_params.culprit_tool
 
         interval_date = datetime.now(timezone.utc) - timedelta(days=interval_in_days)
         interval_param = interval_date.replace(
             hour=0, minute=0, second=0, microsecond=0
         )
 
         self.issue_records = get_issue_listing_data(
-            origin=origin_param, interval_date=interval_param
+            interval_date=interval_param,
+            culprit_code=culprit_code,
+            culprit_harness=culprit_harness,
+            culprit_tool=culprit_tool,
         )
 
         if len(self.issue_records) == 0:

backend/requests/issue-listing-get.sh

@@ -1,4 +1,4 @@
-http 'localhost:8000/api/issue/?origin=maestro&intervalInDays=1'
+http 'localhost:8000/api/issue/?&intervalInDays=1&culpritCode=true'
 
 # HTTP/1.1 200 OK
 # Allow: GET, HEAD, OPTIONS
@@ -15,61 +15,63 @@ http 'localhost:8000/api/issue/?origin=maestro&intervalInDays=1'
 # X-Frame-Options: DENY
 
 # {
-#     "extras": {
-#         "maestro:031929b615e8ecfed14a89cef2b564108c3aa42c": {
-#             "first_seen": "2025-02-17T20:40:01.251254Z",
-#             "git_commit_hash": "a1c2670b1577474f7ae60d5dec1d35a053d354f0",
-#             "git_commit_name": "v6.1.124-16535-ga1c2670b1577",
-#             "git_repository_branch": "chromeos-6.1",
-#             "git_repository_url": "https://chromium.googlesource.com/chromiumos/third_party/kernel",
-#             "tree_name": "chromiumos"
-#         },
-#         "maestro:069157741b947b2f589c971be301429c4bb72515": {
-#             "first_seen": "2025-02-17T20:40:01.251254Z",
-#             "git_commit_hash": "a1c2670b1577474f7ae60d5dec1d35a053d354f0",
-#             "git_commit_name": "v6.1.124-16535-ga1c2670b1577",
-#             "git_repository_branch": "chromeos-6.1",
-#             "git_repository_url": "https://chromium.googlesource.com/chromiumos/third_party/kernel",
-#             "tree_name": "chromiumos"
-#         },
-#         "maestro:11eb365e0d1a4de6c80b036cd2e7a3fe5c276587": {
-#             "first_seen": "2024-11-12T10:30:20.456672Z",
-#             "git_commit_hash": "d145d3aa80067e115a679d903fba256c3d1f39a1",
-#             "git_commit_name": "v5.4.285-53-gd145d3aa8006",
-#             "git_repository_branch": "linux-5.4.y",
-#             "git_repository_url": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git",
-#             "tree_name": "stable-rc"
-#         },
-#         ...
-#     },
-#     "issues": [
-#         {
-#             "comment": " call to undeclared function 'stack_trace_save_tsk'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] in kernel/sched/core.o (kernel/sched/core.c) [logspec:kbuild,kbuild.compiler.error]",
-#             "culprit_code": true,
-#             "culprit_harness": false,
-#             "culprit_tool": false,
-#             "field_timestamp": "2025-02-19T22:51:01.778279Z",
-#             "id": "maestro:87244933628a2612f39e6096115454f1e8bb3e1c",
-#             "version": 1
-#         },
-#         {
-#             "comment": " implicit declaration of function ‘stack_trace_save_tsk’ [-Werror=implicit-function-declaration] in kernel/sched/core.o (kernel/sched/core.c) [logspec:kbuild,kbuild.compiler.error]",
-#             "culprit_code": true,
-#             "culprit_harness": false,
-#             "culprit_tool": false,
-#             "field_timestamp": "2025-02-19T23:42:17.982170Z",
-#             "id": "maestro:2ff8fe94f6d53f39321d4a37fe15801cedc93573",
-#             "version": 1
-#         },
-#         {
-#             "comment": " implicit declaration of function 'drm_connector_helper_hpd_irq_event' [-Werror,-Wimplicit-function-declaration] in drivers/gpu/drm/rockchip/cdn-dp-core.o (drivers/gpu/drm/rockchip/cdn-dp-core.c) [logspec:kbuild,kbuild.compiler.error]",
-#             "culprit_code": true,
-#             "culprit_harness": false,
-#             "culprit_tool": false,
-#             "field_timestamp": "2025-02-20T11:50:02.465668Z",
-#             "id": "maestro:3feccbb8dd7c7976251cf4457318fc92c3eb2efb",
-#             "version": 1
-#         },
-#         ...
-#     ]
-# }
+#     "issues":[
+#        {
+#           "field_timestamp":"2025-03-21T05:50:33.654300Z",
+#           "id":"maestro:cd18b383b75d152345cbe08983013da942678433",
+#           "comment":" NULL pointer dereference at virtual address 0000000000000030 [logspec:generic_linux_boot,linux.kernel.null_pointer_dereference]",
+#           "origin":"maestro",
+#           "version":1,
+#           "culprit_code":true,
+#           "culprit_tool":false,
+#           "culprit_harness":false
+#        },
+#        {
+#           "field_timestamp":"2025-03-20T19:02:01.275697Z",
+#           "id":"redhat:issue_3652",
+#           "comment":"networking/netfilter/upstream_test/nftables  upstream-binary-nftables-tests-shell kernel warning net/core/flow_dissector.c",
+#           "origin":"redhat",
+#           "version":1742488543,
+#           "culprit_code":true,
+#           "culprit_tool":false,
+#           "culprit_harness":false
+#        },
+#        {
+#           "field_timestamp":"2025-03-21T11:22:57.073931Z",
+#           "id":"redhat:issue_3585",
+#           "comment":"[RHEL10] [internal-testsuite] perf-pipe-recording-and-injection-test FAIL",
+#           "origin":"redhat",
+#           "version":1741311124,
+#           "culprit_code":true,
+#           "culprit_tool":false,
+#           "culprit_harness":false
+#        },
+
+#     ],
+#     "extras":{
+#        "redhat:issue_3492":{
+#           "first_seen":"2025-02-12T13:00:01.018238Z",
+#           "git_commit_hash":null,
+#           "git_repository_url":null,
+#           "git_repository_branch":null,
+#           "git_commit_name":null,
+#           "tree_name":null
+#        },
+#        "maestro:e602fca280d85d8e603f7c0aff68363bb0cd7993":{
+#           "first_seen":"2025-01-21T00:24:05.705873Z",
+#           "git_commit_hash":"d73a4602e973e9e922f00c537a4643907a547ade",
+#           "git_repository_url":"https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git",
+#           "git_repository_branch":"main",
+#           "git_commit_name":"pm-6.13-rc8-1598-gd73a4602e973",
+#           "tree_name":"net-next"
+#        },
+#        "maestro:da694c56147298d223ee432ad8d6a8ee311b773a":{
+#           "first_seen":"2025-01-21T00:22:10.827866Z",
+#           "git_commit_hash":"d73a4602e973e9e922f00c537a4643907a547ade",
+#           "git_repository_url":"https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git",
+#           "git_repository_branch":"main",
+#           "git_commit_name":"pm-6.13-rc8-1598-gd73a4602e973",
+#           "tree_name":"net-next"
+#        }
+#     }
+#  }