From 7165e743e9d9cf11a86403fdd799589731d86c5b Mon Sep 17 00:00:00 2001 From: pszulczewski Date: Tue, 18 Apr 2023 16:46:27 +0200 Subject: [PATCH 1/3] Base-line tests for future refactor. --- .../napalm_get_bgp_neighbors/multi_vrf.json | 138 ++++++++++++++++++ tests/test_get_value.py | 56 ++++++- 2 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 tests/mock/napalm_get_bgp_neighbors/multi_vrf.json diff --git a/tests/mock/napalm_get_bgp_neighbors/multi_vrf.json b/tests/mock/napalm_get_bgp_neighbors/multi_vrf.json new file mode 100644 index 0000000..0064472 --- /dev/null +++ b/tests/mock/napalm_get_bgp_neighbors/multi_vrf.json @@ -0,0 +1,138 @@ +{ + "global": { + "peers": { + "10.1.0.0": { + "address_family": { + "ipv4": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + }, + "ipv6": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + } + }, + "description": "", + "is_enabled": true, + "is_up": true, + "local_as": 4268360780, + "remote_as": 67890, + "remote_id": "0.0.0.0", + "uptime": 1783 + }, + "10.2.0.0": { + "address_family": { + "ipv4": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + }, + "ipv6": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + } + }, + "description": "", + "is_enabled": true, + "is_up": false, + "local_as": 4268360780, + "remote_as": 67890, + "remote_id": "0.0.0.0", + "uptime": 1782 + }, + "10.64.207.255": { + "address_family": { + "ipv4": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + }, + "ipv6": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + } + }, + "description": "", + "is_enabled": false, + "is_up": true, + "local_as": 4268360780, + "remote_as": 12345, + "remote_id": "0.0.0.0", + "uptime": 1782 + }, + "7.7.7.7": { + "address_family": { + "ipv4": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + }, + "ipv6": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + } + }, + "description": "", + "is_enabled": true, + "is_up": true, + "local_as": 4268360780, + "remote_as": 67890, + "remote_id": "0.0.0.0", + "uptime": 0 + } + }, + "router_id": "1.1.0.1" + }, + "vpn": { + "peers": { + "10.1.0.0": { + "address_family": { + "ipv4": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + }, + "ipv6": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + } + }, + "description": "", + "is_enabled": true, + "is_up": true, + "local_as": 4268360780, + "remote_as": 67890, + "remote_id": "0.0.0.0", + "uptime": 1783 + }, + "10.2.0.0": { + "address_family": { + "ipv4": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + }, + "ipv6": { + "accepted_prefixes": 1000, + "received_prefixes": 1000, + "sent_prefixes": 1000 + } + }, + "description": "", + "is_enabled": true, + "is_up": false, + "local_as": 4268360780, + "remote_as": 67890, + "remote_id": "0.0.0.0", + "uptime": 1782 + } + }, + "router_id": "10.1.0.1" + } +} diff --git a/tests/test_get_value.py b/tests/test_get_value.py index 737a12a..351c23a 100644 --- a/tests/test_get_value.py +++ b/tests/test_get_value.py @@ -1,16 +1,58 @@ -"""Test GitHub issues.""" +"""Test extract_data_from_json.""" import pytest from jdiff import extract_data_from_json +from .utility import load_json_file -my_data = [{"global": {"peers": {"10.1.0.0": "peer1", "10.2.0.0": "peer2"}}}] +test_cases_extract_data_none = [ + "global[*]", + 'global.peers."1.1.1.1"', +] -@pytest.mark.parametrize("data", my_data) -def test_jmspath_return_none(data): - """Handle exception when JMSPath retunr None.""" - my_jmspath = "global[*]" +@pytest.mark.parametrize("jmspath", test_cases_extract_data_none) +def test_jmspath_return_none(jmspath): + """Handle exception when JMSPath returns None.""" + data = {"global": {"peers": {"10.1.0.0": "peer1", "10.2.0.0": "peer2"}}} with pytest.raises(TypeError) as error: - extract_data_from_json(data=data, path=my_jmspath)() # pylint: disable=E0110 + extract_data_from_json(data=data, path=jmspath) assert "JMSPath returned 'None'. Please, verify your JMSPath regex." in error.value.__str__() + + +test_cases_extract_data_no_ref_key = [ + ("global.peers.*.*.ipv6.[accepted_prefixes]", [[1000], [1000], [1000], [1000]]), + ("vpn.peers.*.*.ipv6.[accepted_prefixes]", [[1000], [1000]]), + ( + "*.peers.*.*.*.[accepted_prefixes]", + [[1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000]], + ), +] + + +test_cases_extract_data__with_ref_key = [ + ( + "global.peers.$*$.*.ipv6.[accepted_prefixes]", + [ + {"10.1.0.0": {"accepted_prefixes": 1000}}, + {"10.2.0.0": {"accepted_prefixes": 1000}}, + {"10.64.207.255": {"accepted_prefixes": 1000}}, + {"7.7.7.7": {"accepted_prefixes": 1000}}, + ], + ), + ( + "vpn.peers.$*$.*.ipv6.[accepted_prefixes]", + [{"10.1.0.0": {"accepted_prefixes": 1000}}, {"10.2.0.0": {"accepted_prefixes": 1000}}], + ), +] + + +@pytest.mark.parametrize( + "jmspath, expected_value", test_cases_extract_data_no_ref_key + test_cases_extract_data__with_ref_key +) +def test_extract_data_from_json(jmspath, expected_value): + """Test JMSPath return value.""" + data = load_json_file("napalm_get_bgp_neighbors", "multi_vrf.json") + value = extract_data_from_json(data=data, path=jmspath) + + assert value == expected_value From 4aafeebbe8e4fe36f975de053ae31597325096e2 Mon Sep 17 00:00:00 2001 From: pszulczewski Date: Tue, 18 Apr 2023 18:33:05 +0200 Subject: [PATCH 2/3] Refactor to simplify code. --- jdiff/utils/jmespath_parsers.py | 6 +++--- tests/test_get_value.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jdiff/utils/jmespath_parsers.py b/jdiff/utils/jmespath_parsers.py index 34f7777..126a20b 100644 --- a/jdiff/utils/jmespath_parsers.py +++ b/jdiff/utils/jmespath_parsers.py @@ -31,15 +31,15 @@ def jmespath_value_parser(path: str): path_suffix = path.split(".")[-1] if regex_match_ref_key: + reference_key = regex_match_ref_key.group() if regex_ref_key.search(path_suffix): # [$peerAddress$,prefixesReceived] --> [prefixesReceived] - reference_key = regex_match_ref_key.group() return path.replace(reference_key, "") # result[0].$vrfs$.default... --> result[0].vrfs.default.... - regex_normalized_value = re.search(r"\$.*\$", regex_match_ref_key.group()) + regex_normalized_value = re.search(r"\$.*\$", reference_key) if regex_normalized_value: - normalized_value = regex_match_ref_key.group().split("$")[1] + normalized_value = reference_key.split("$")[1] return path.replace(regex_normalized_value.group(), normalized_value) return path diff --git a/tests/test_get_value.py b/tests/test_get_value.py index 351c23a..658e9ed 100644 --- a/tests/test_get_value.py +++ b/tests/test_get_value.py @@ -30,7 +30,7 @@ def test_jmspath_return_none(jmspath): ] -test_cases_extract_data__with_ref_key = [ +test_cases_extract_data_with_ref_key = [ ( "global.peers.$*$.*.ipv6.[accepted_prefixes]", [ @@ -48,7 +48,7 @@ def test_jmspath_return_none(jmspath): @pytest.mark.parametrize( - "jmspath, expected_value", test_cases_extract_data_no_ref_key + test_cases_extract_data__with_ref_key + "jmspath, expected_value", test_cases_extract_data_no_ref_key + test_cases_extract_data_with_ref_key ) def test_extract_data_from_json(jmspath, expected_value): """Test JMSPath return value.""" From c8b784afd76d4c0be95fcb7f8a42098d2d62a37c Mon Sep 17 00:00:00 2001 From: pszulczewski Date: Wed, 19 Apr 2023 23:22:42 +0200 Subject: [PATCH 3/3] Implement issue #92 --- jdiff/extract_data.py | 7 ++++++ jdiff/utils/jmespath_parsers.py | 39 +++++++++++++++++++++++++++++++ tests/test_get_value.py | 31 +++++++++++++++++++++++-- tests/test_jmespath_parsers.py | 41 ++++++++++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/jdiff/extract_data.py b/jdiff/extract_data.py index 1311970..f2b951a 100644 --- a/jdiff/extract_data.py +++ b/jdiff/extract_data.py @@ -9,6 +9,7 @@ jmespath_refkey_parser, associate_key_of_my_value, keys_values_zipper, + multi_reference_keys, ) @@ -43,6 +44,12 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude: # return if path is not specified return data + # Multi ref_key + if len(re.findall(r"\$.*?\$", path)) > 1: + clean_path = path.replace("$", "") + values = jmespath.search(f"{clean_path}{' | []' * (path.count('*') - 1)}", data) + return keys_values_zipper(multi_reference_keys(path, data), associate_key_of_my_value(clean_path, values)) + values = jmespath.search(jmespath_value_parser(path), data) if values is None: diff --git a/jdiff/utils/jmespath_parsers.py b/jdiff/utils/jmespath_parsers.py index 126a20b..2cb88b8 100644 --- a/jdiff/utils/jmespath_parsers.py +++ b/jdiff/utils/jmespath_parsers.py @@ -8,6 +8,8 @@ import re from typing import Mapping, List, Union +import jmespath + def jmespath_value_parser(path: str): """ @@ -120,3 +122,40 @@ def keys_values_zipper(list_of_reference_keys: List, wanted_value_with_key: List final_result.append({my_key: wanted_value_with_key[my_index]}) return final_result + + +def multi_reference_keys(jmspath, data): + """Build a list of concatenated reference keys. + + Args: + jmspath: "$*$.peers.$*$.*.ipv4.[accepted_prefixes]" + data: tests/mock/napalm_get_bgp_neighbors/multi_vrf.json + + Returns: + ["global.10.1.0.0", "global.10.2.0.0", "global.10.64.207.255", "global.7.7.7.7", "vpn.10.1.0.0", "vpn.10.2.0.0"] + """ + ref_key_regex = re.compile(r"\$.*?\$") + mapping = [] + split_path = jmspath.split(".") + + ref_key_index = -1 # -1 as the starting value, so it will match split path list indexes + for index, element in enumerate(split_path): + if ref_key_regex.search(element): + ref_key_index += 1 + key_path = ( + ".".join(split_path[:index]).replace("$", "") or "@" + ) # @ is for top keys, as they are stripped with "*" + flat_path = f"{key_path}{' | []' * key_path.count('*')}" # | [] to flatten the data, nesting level is eq to "*" count + sub_data = jmespath.search(flat_path, data) # extract sub-data with up to the ref key + if isinstance(sub_data, dict): + keys = list(sub_data.keys()) + elif isinstance(sub_data, list): + keys = [] + for parent, children in zip( + mapping[ref_key_index - 1], sub_data + ): # refer to previous keys as they are already present in mapping + keys.extend(f"{parent}.{child}" for child in children.keys()) # concatenate keys + else: + raise ValueError("Ref key anchor must return either a dict or a list.") + mapping.append(keys) + return mapping[-1] # return last element as it has all previous ref_keys concatenated. diff --git a/tests/test_get_value.py b/tests/test_get_value.py index 658e9ed..10200b1 100644 --- a/tests/test_get_value.py +++ b/tests/test_get_value.py @@ -1,7 +1,7 @@ """Test extract_data_from_json.""" import pytest from jdiff import extract_data_from_json -from .utility import load_json_file +from .utility import load_json_file, ASSERT_FAIL_MESSAGE test_cases_extract_data_none = [ @@ -44,6 +44,33 @@ def test_jmspath_return_none(jmspath): "vpn.peers.$*$.*.ipv6.[accepted_prefixes]", [{"10.1.0.0": {"accepted_prefixes": 1000}}, {"10.2.0.0": {"accepted_prefixes": 1000}}], ), + ( + "$*$.peers.$*$.*.ipv4.[accepted_prefixes,received_prefixes,sent_prefixes]", + [ + {"global.10.1.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}}, + {"global.10.2.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}}, + {"global.10.64.207.255": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}}, + {"global.7.7.7.7": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}}, + {"vpn.10.1.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}}, + {"vpn.10.2.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}}, + ], + ), + ( + "$*$.peers.$*$.*.ipv6.[received_prefixes,sent_prefixes]", + [ + {"global.10.1.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}}, + {"global.10.2.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}}, + {"global.10.64.207.255": {"received_prefixes": 1000, "sent_prefixes": 1000}}, + {"global.7.7.7.7": {"received_prefixes": 1000, "sent_prefixes": 1000}}, + {"vpn.10.1.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}}, + {"vpn.10.2.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}}, + ], + ), + pytest.param( + "$*$.peers.$*$.address_family.$*$.[accepted_prefixes]", + "", + marks=pytest.mark.xfail(reason="Jmespath issue - path returns empty list."), + ), ] @@ -55,4 +82,4 @@ def test_extract_data_from_json(jmspath, expected_value): data = load_json_file("napalm_get_bgp_neighbors", "multi_vrf.json") value = extract_data_from_json(data=data, path=jmspath) - assert value == expected_value + assert value == expected_value, ASSERT_FAIL_MESSAGE.format(output=value, expected_output=expected_value) diff --git a/tests/test_jmespath_parsers.py b/tests/test_jmespath_parsers.py index a3c64de..972acc1 100644 --- a/tests/test_jmespath_parsers.py +++ b/tests/test_jmespath_parsers.py @@ -5,8 +5,9 @@ jmespath_refkey_parser, keys_values_zipper, associate_key_of_my_value, + multi_reference_keys, ) -from .utility import ASSERT_FAIL_MESSAGE +from .utility import load_json_file, ASSERT_FAIL_MESSAGE value_parser_case_1 = ( @@ -112,3 +113,41 @@ def test_keys_zipper(ref_keys, wanted_values, expected_output): def test_keys_association(path, wanted_values, expected_output): output = associate_key_of_my_value(path, wanted_values) assert expected_output == output, ASSERT_FAIL_MESSAGE.format(output=output, expected_output=expected_output) + + +multi_ref_key_case_1 = ( + "$*$.peers.$*$.*.ipv4.[accepted_prefixes]", + ["global.10.1.0.0", "global.10.2.0.0", "global.10.64.207.255", "global.7.7.7.7", "vpn.10.1.0.0", "vpn.10.2.0.0"], +) + + +multi_ref_key_case_2 = ( + "$*$.peers.$*$.address_family.$*$.[accepted_prefixes]", + [ + "global.10.1.0.0.ipv4", + "global.10.1.0.0.ipv6", + "global.10.2.0.0.ipv4", + "global.10.2.0.0.ipv6", + "global.10.64.207.255.ipv4", + "global.10.64.207.255.ipv6", + "global.7.7.7.7.ipv4", + "global.7.7.7.7.ipv6", + "vpn.10.1.0.0.ipv4", + "vpn.10.1.0.0.ipv6", + "vpn.10.2.0.0.ipv4", + "vpn.10.2.0.0.ipv6", + ], +) + + +multi_ref_key_test_cases = [ + multi_ref_key_case_1, + multi_ref_key_case_2, +] + + +@pytest.mark.parametrize("path, expected_output", multi_ref_key_test_cases) +def test_multi_ref_key(path, expected_output): + data = load_json_file("napalm_get_bgp_neighbors", "multi_vrf.json") + output = multi_reference_keys(path, data) + assert expected_output == output, ASSERT_FAIL_MESSAGE.format(output=output, expected_output=expected_output)