Skip to content

Commit b1af665

Browse files
pszulczewskipszulczewski
authored
Issue 92 (#98)
* Base-line tests for future refactor. * Refactor to simplify code. * Implement issue #92
1 parent e8280c7 commit b1af665

File tree

5 files changed

+303
-11
lines changed

5 files changed

+303
-11
lines changed

‎jdiff/extract_data.py

+7
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
jmespath_refkey_parser,
1010
associate_key_of_my_value,
1111
keys_values_zipper,
12+
multi_reference_keys,
1213
)
1314

1415

@@ -43,6 +44,12 @@ def extract_data_from_json(data: Union[Mapping, List], path: str = "*", exclude:
4344
# return if path is not specified
4445
return data
4546

47+
# Multi ref_key
48+
if len(re.findall(r"\$.*?\$", path)) > 1:
49+
clean_path = path.replace("$", "")
50+
values = jmespath.search(f"{clean_path}{' | []' * (path.count('*') - 1)}", data)
51+
return keys_values_zipper(multi_reference_keys(path, data), associate_key_of_my_value(clean_path, values))
52+
4653
values = jmespath.search(jmespath_value_parser(path), data)
4754

4855
if values is None:

‎jdiff/utils/jmespath_parsers.py

+42-3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import re
99
from typing import Mapping, List, Union
1010

11+
import jmespath
12+
1113

1214
def jmespath_value_parser(path: str):
1315
"""
@@ -31,15 +33,15 @@ def jmespath_value_parser(path: str):
3133
path_suffix = path.split(".")[-1]
3234

3335
if regex_match_ref_key:
36+
reference_key = regex_match_ref_key.group()
3437
if regex_ref_key.search(path_suffix):
3538
# [$peerAddress$,prefixesReceived] --> [prefixesReceived]
36-
reference_key = regex_match_ref_key.group()
3739
return path.replace(reference_key, "")
3840

3941
# result[0].$vrfs$.default... --> result[0].vrfs.default....
40-
regex_normalized_value = re.search(r"\$.*\$", regex_match_ref_key.group())
42+
regex_normalized_value = re.search(r"\$.*\$", reference_key)
4143
if regex_normalized_value:
42-
normalized_value = regex_match_ref_key.group().split("$")[1]
44+
normalized_value = reference_key.split("$")[1]
4345
return path.replace(regex_normalized_value.group(), normalized_value)
4446
return path
4547

@@ -120,3 +122,40 @@ def keys_values_zipper(list_of_reference_keys: List, wanted_value_with_key: List
120122
final_result.append({my_key: wanted_value_with_key[my_index]})
121123

122124
return final_result
125+
126+
127+
def multi_reference_keys(jmspath, data):
128+
"""Build a list of concatenated reference keys.
129+
130+
Args:
131+
jmspath: "$*$.peers.$*$.*.ipv4.[accepted_prefixes]"
132+
data: tests/mock/napalm_get_bgp_neighbors/multi_vrf.json
133+
134+
Returns:
135+
["global.10.1.0.0", "global.10.2.0.0", "global.10.64.207.255", "global.7.7.7.7", "vpn.10.1.0.0", "vpn.10.2.0.0"]
136+
"""
137+
ref_key_regex = re.compile(r"\$.*?\$")
138+
mapping = []
139+
split_path = jmspath.split(".")
140+
141+
ref_key_index = -1 # -1 as the starting value, so it will match split path list indexes
142+
for index, element in enumerate(split_path):
143+
if ref_key_regex.search(element):
144+
ref_key_index += 1
145+
key_path = (
146+
".".join(split_path[:index]).replace("$", "") or "@"
147+
) # @ is for top keys, as they are stripped with "*"
148+
flat_path = f"{key_path}{' | []' * key_path.count('*')}" # | [] to flatten the data, nesting level is eq to "*" count
149+
sub_data = jmespath.search(flat_path, data) # extract sub-data with up to the ref key
150+
if isinstance(sub_data, dict):
151+
keys = list(sub_data.keys())
152+
elif isinstance(sub_data, list):
153+
keys = []
154+
for parent, children in zip(
155+
mapping[ref_key_index - 1], sub_data
156+
): # refer to previous keys as they are already present in mapping
157+
keys.extend(f"{parent}.{child}" for child in children.keys()) # concatenate keys
158+
else:
159+
raise ValueError("Ref key anchor must return either a dict or a list.")
160+
mapping.append(keys)
161+
return mapping[-1] # return last element as it has all previous ref_keys concatenated.

‎tests/mock/napalm_get_bgp_neighbors/multi_vrf.json

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
{
2+
"global": {
3+
"peers": {
4+
"10.1.0.0": {
5+
"address_family": {
6+
"ipv4": {
7+
"accepted_prefixes": 1000,
8+
"received_prefixes": 1000,
9+
"sent_prefixes": 1000
10+
},
11+
"ipv6": {
12+
"accepted_prefixes": 1000,
13+
"received_prefixes": 1000,
14+
"sent_prefixes": 1000
15+
}
16+
},
17+
"description": "",
18+
"is_enabled": true,
19+
"is_up": true,
20+
"local_as": 4268360780,
21+
"remote_as": 67890,
22+
"remote_id": "0.0.0.0",
23+
"uptime": 1783
24+
},
25+
"10.2.0.0": {
26+
"address_family": {
27+
"ipv4": {
28+
"accepted_prefixes": 1000,
29+
"received_prefixes": 1000,
30+
"sent_prefixes": 1000
31+
},
32+
"ipv6": {
33+
"accepted_prefixes": 1000,
34+
"received_prefixes": 1000,
35+
"sent_prefixes": 1000
36+
}
37+
},
38+
"description": "",
39+
"is_enabled": true,
40+
"is_up": false,
41+
"local_as": 4268360780,
42+
"remote_as": 67890,
43+
"remote_id": "0.0.0.0",
44+
"uptime": 1782
45+
},
46+
"10.64.207.255": {
47+
"address_family": {
48+
"ipv4": {
49+
"accepted_prefixes": 1000,
50+
"received_prefixes": 1000,
51+
"sent_prefixes": 1000
52+
},
53+
"ipv6": {
54+
"accepted_prefixes": 1000,
55+
"received_prefixes": 1000,
56+
"sent_prefixes": 1000
57+
}
58+
},
59+
"description": "",
60+
"is_enabled": false,
61+
"is_up": true,
62+
"local_as": 4268360780,
63+
"remote_as": 12345,
64+
"remote_id": "0.0.0.0",
65+
"uptime": 1782
66+
},
67+
"7.7.7.7": {
68+
"address_family": {
69+
"ipv4": {
70+
"accepted_prefixes": 1000,
71+
"received_prefixes": 1000,
72+
"sent_prefixes": 1000
73+
},
74+
"ipv6": {
75+
"accepted_prefixes": 1000,
76+
"received_prefixes": 1000,
77+
"sent_prefixes": 1000
78+
}
79+
},
80+
"description": "",
81+
"is_enabled": true,
82+
"is_up": true,
83+
"local_as": 4268360780,
84+
"remote_as": 67890,
85+
"remote_id": "0.0.0.0",
86+
"uptime": 0
87+
}
88+
},
89+
"router_id": "1.1.0.1"
90+
},
91+
"vpn": {
92+
"peers": {
93+
"10.1.0.0": {
94+
"address_family": {
95+
"ipv4": {
96+
"accepted_prefixes": 1000,
97+
"received_prefixes": 1000,
98+
"sent_prefixes": 1000
99+
},
100+
"ipv6": {
101+
"accepted_prefixes": 1000,
102+
"received_prefixes": 1000,
103+
"sent_prefixes": 1000
104+
}
105+
},
106+
"description": "",
107+
"is_enabled": true,
108+
"is_up": true,
109+
"local_as": 4268360780,
110+
"remote_as": 67890,
111+
"remote_id": "0.0.0.0",
112+
"uptime": 1783
113+
},
114+
"10.2.0.0": {
115+
"address_family": {
116+
"ipv4": {
117+
"accepted_prefixes": 1000,
118+
"received_prefixes": 1000,
119+
"sent_prefixes": 1000
120+
},
121+
"ipv6": {
122+
"accepted_prefixes": 1000,
123+
"received_prefixes": 1000,
124+
"sent_prefixes": 1000
125+
}
126+
},
127+
"description": "",
128+
"is_enabled": true,
129+
"is_up": false,
130+
"local_as": 4268360780,
131+
"remote_as": 67890,
132+
"remote_id": "0.0.0.0",
133+
"uptime": 1782
134+
}
135+
},
136+
"router_id": "10.1.0.1"
137+
}
138+
}

‎tests/test_get_value.py

+76-7
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,85 @@
1-
"""Test GitHub issues."""
1+
"""Test extract_data_from_json."""
22
import pytest
33
from jdiff import extract_data_from_json
4+
from .utility import load_json_file, ASSERT_FAIL_MESSAGE
45

56

6-
my_data = [{"global": {"peers": {"10.1.0.0": "peer1", "10.2.0.0": "peer2"}}}]
7+
test_cases_extract_data_none = [
8+
"global[*]",
9+
'global.peers."1.1.1.1"',
10+
]
711

812

9-
@pytest.mark.parametrize("data", my_data)
10-
def test_jmspath_return_none(data):
11-
"""Handle exception when JMSPath retunr None."""
12-
my_jmspath = "global[*]"
13+
@pytest.mark.parametrize("jmspath", test_cases_extract_data_none)
14+
def test_jmspath_return_none(jmspath):
15+
"""Handle exception when JMSPath returns None."""
16+
data = {"global": {"peers": {"10.1.0.0": "peer1", "10.2.0.0": "peer2"}}}
1317
with pytest.raises(TypeError) as error:
14-
extract_data_from_json(data=data, path=my_jmspath)() # pylint: disable=E0110
18+
extract_data_from_json(data=data, path=jmspath)
1519

1620
assert "JMSPath returned 'None'. Please, verify your JMSPath regex." in error.value.__str__()
21+
22+
23+
test_cases_extract_data_no_ref_key = [
24+
("global.peers.*.*.ipv6.[accepted_prefixes]", [[1000], [1000], [1000], [1000]]),
25+
("vpn.peers.*.*.ipv6.[accepted_prefixes]", [[1000], [1000]]),
26+
(
27+
"*.peers.*.*.*.[accepted_prefixes]",
28+
[[1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000], [1000]],
29+
),
30+
]
31+
32+
33+
test_cases_extract_data_with_ref_key = [
34+
(
35+
"global.peers.$*$.*.ipv6.[accepted_prefixes]",
36+
[
37+
{"10.1.0.0": {"accepted_prefixes": 1000}},
38+
{"10.2.0.0": {"accepted_prefixes": 1000}},
39+
{"10.64.207.255": {"accepted_prefixes": 1000}},
40+
{"7.7.7.7": {"accepted_prefixes": 1000}},
41+
],
42+
),
43+
(
44+
"vpn.peers.$*$.*.ipv6.[accepted_prefixes]",
45+
[{"10.1.0.0": {"accepted_prefixes": 1000}}, {"10.2.0.0": {"accepted_prefixes": 1000}}],
46+
),
47+
(
48+
"$*$.peers.$*$.*.ipv4.[accepted_prefixes,received_prefixes,sent_prefixes]",
49+
[
50+
{"global.10.1.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}},
51+
{"global.10.2.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}},
52+
{"global.10.64.207.255": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}},
53+
{"global.7.7.7.7": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}},
54+
{"vpn.10.1.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}},
55+
{"vpn.10.2.0.0": {"accepted_prefixes": 1000, "received_prefixes": 1000, "sent_prefixes": 1000}},
56+
],
57+
),
58+
(
59+
"$*$.peers.$*$.*.ipv6.[received_prefixes,sent_prefixes]",
60+
[
61+
{"global.10.1.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}},
62+
{"global.10.2.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}},
63+
{"global.10.64.207.255": {"received_prefixes": 1000, "sent_prefixes": 1000}},
64+
{"global.7.7.7.7": {"received_prefixes": 1000, "sent_prefixes": 1000}},
65+
{"vpn.10.1.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}},
66+
{"vpn.10.2.0.0": {"received_prefixes": 1000, "sent_prefixes": 1000}},
67+
],
68+
),
69+
pytest.param(
70+
"$*$.peers.$*$.address_family.$*$.[accepted_prefixes]",
71+
"",
72+
marks=pytest.mark.xfail(reason="Jmespath issue - path returns empty list."),
73+
),
74+
]
75+
76+
77+
@pytest.mark.parametrize(
78+
"jmspath, expected_value", test_cases_extract_data_no_ref_key + test_cases_extract_data_with_ref_key
79+
)
80+
def test_extract_data_from_json(jmspath, expected_value):
81+
"""Test JMSPath return value."""
82+
data = load_json_file("napalm_get_bgp_neighbors", "multi_vrf.json")
83+
value = extract_data_from_json(data=data, path=jmspath)
84+
85+
assert value == expected_value, ASSERT_FAIL_MESSAGE.format(output=value, expected_output=expected_value)

‎tests/test_jmespath_parsers.py

+40-1
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@
55
jmespath_refkey_parser,
66
keys_values_zipper,
77
associate_key_of_my_value,
8+
multi_reference_keys,
89
)
9-
from .utility import ASSERT_FAIL_MESSAGE
10+
from .utility import load_json_file, ASSERT_FAIL_MESSAGE
1011

1112

1213
value_parser_case_1 = (
@@ -112,3 +113,41 @@ def test_keys_zipper(ref_keys, wanted_values, expected_output):
112113
def test_keys_association(path, wanted_values, expected_output):
113114
output = associate_key_of_my_value(path, wanted_values)
114115
assert expected_output == output, ASSERT_FAIL_MESSAGE.format(output=output, expected_output=expected_output)
116+
117+
118+
multi_ref_key_case_1 = (
119+
"$*$.peers.$*$.*.ipv4.[accepted_prefixes]",
120+
["global.10.1.0.0", "global.10.2.0.0", "global.10.64.207.255", "global.7.7.7.7", "vpn.10.1.0.0", "vpn.10.2.0.0"],
121+
)
122+
123+
124+
multi_ref_key_case_2 = (
125+
"$*$.peers.$*$.address_family.$*$.[accepted_prefixes]",
126+
[
127+
"global.10.1.0.0.ipv4",
128+
"global.10.1.0.0.ipv6",
129+
"global.10.2.0.0.ipv4",
130+
"global.10.2.0.0.ipv6",
131+
"global.10.64.207.255.ipv4",
132+
"global.10.64.207.255.ipv6",
133+
"global.7.7.7.7.ipv4",
134+
"global.7.7.7.7.ipv6",
135+
"vpn.10.1.0.0.ipv4",
136+
"vpn.10.1.0.0.ipv6",
137+
"vpn.10.2.0.0.ipv4",
138+
"vpn.10.2.0.0.ipv6",
139+
],
140+
)
141+
142+
143+
multi_ref_key_test_cases = [
144+
multi_ref_key_case_1,
145+
multi_ref_key_case_2,
146+
]
147+
148+
149+
@pytest.mark.parametrize("path, expected_output", multi_ref_key_test_cases)
150+
def test_multi_ref_key(path, expected_output):
151+
data = load_json_file("napalm_get_bgp_neighbors", "multi_vrf.json")
152+
output = multi_reference_keys(path, data)
153+
assert expected_output == output, ASSERT_FAIL_MESSAGE.format(output=output, expected_output=expected_output)

0 commit comments

Comments
 (0)