From 575a1c91d22f16ddb082049f4f2728fa3f1c763e Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Sun, 3 Sep 2023 22:14:17 +0900 Subject: [PATCH 1/7] add function to combine accumulator update --- pythclient/price_feeds.py | 213 ++++++++++++++++++++++++++++++++------ 1 file changed, 183 insertions(+), 30 deletions(-) diff --git a/pythclient/price_feeds.py b/pythclient/price_feeds.py index 10c4d3c..f83acbf 100644 --- a/pythclient/price_feeds.py +++ b/pythclient/price_feeds.py @@ -1,5 +1,6 @@ import base64 import binascii +import struct from struct import unpack from typing import Any, Dict, List, Optional @@ -15,6 +16,8 @@ ACCUMULATOR_MAGIC = "504e4155" +MAX_MESSAGE_IN_SINGLE_UPDATE_DATA = 255 + class Price: def __init__(self, conf, expo, price, publish_time): @@ -412,7 +415,7 @@ def price_attestation_to_price_feed(price_attestation): # Referenced from https://github.com/pyth-network/pyth-crosschain/blob/1a00598334e52fc5faf967eb1170d7fc23ad828b/price_service/server/src/rest.ts#L137 def extract_price_info_from_accumulator_update( - price_feed_id, update_data, encoding + update_data, encoding ) -> Optional[Dict[str, Any]]: encoded_update_data = encode_vaa_for_chain(update_data, encoding, buffer=True) offset = 0 @@ -447,18 +450,26 @@ def extract_price_info_from_accumulator_update( num_updates = encoded_update_data[offset] offset += 1 + price_infos = [] for _ in range(num_updates): message_length = int.from_bytes( encoded_update_data[offset : offset + 2], byteorder="big" - ) + ) # message_size: u16 offset += 2 - message = encoded_update_data[offset : offset + message_length] + message = encoded_update_data[ + offset : offset + message_length + ] # message: [u8; message_size] offset += message_length - proof_length = encoded_update_data[offset] + proof_length = encoded_update_data[offset] # proof_size: u8 offset += 1 - offset += proof_length # ignore proofs + + proof = [] + for _ in range(proof_length): + hash = encoded_update_data[offset : offset + 20] + proof.append(hash) + offset += 20 message_offset = 0 message_type = message[message_offset] @@ -471,9 +482,6 @@ def extract_price_info_from_accumulator_update( price_id = message[message_offset : message_offset + 32].hex() message_offset += 32 - if price_id != price_feed_id: - continue - price = int.from_bytes( message[message_offset : message_offset + 8], byteorder="big", signed=True ) @@ -502,28 +510,173 @@ def extract_price_info_from_accumulator_update( message[message_offset : message_offset + 8], byteorder="big", signed=False ) - return PriceInfo( - seq_num=parsed_vaa["sequence"], - vaa=update_data, - publish_time=publish_time, - attestation_time=publish_time, - last_attested_publish_time=prev_publish_time, - price_feed=PriceUpdate( - ema_price=Price( - price=str(ema_price), - conf=str(ema_conf), - expo=expo, - publish_time=publish_time, - ), - price_id=price_id, - price=Price( - price=str(price), - conf=str(conf), - expo=expo, - publish_time=publish_time, + price_infos.append( + PriceInfo( + seq_num=parsed_vaa["sequence"], + vaa=update_data, + publish_time=publish_time, + attestation_time=publish_time, + last_attested_publish_time=prev_publish_time, + price_feed=PriceUpdate( + ema_price=Price( + price=str(ema_price), + conf=str(ema_conf), + expo=expo, + publish_time=publish_time, + ), + price_id=price_id, + price=Price( + price=str(price), + conf=str(conf), + expo=expo, + publish_time=publish_time, + ), ), - ), - emitter_chain_id=parsed_vaa["emitter_chain"], + emitter_chain_id=parsed_vaa["emitter_chain"], + ) ) - return None + return price_infos + + +# 1. extract a list of [slot, wh_proof, price_update_message, merkle_proof] per updateData +# 2. combine the ones with the same slot to { same_slot, wh_proof, [price_update_message, merkle_proof] } +# 3. serialize it into the accumulator update format + +# note: you can only merge those with the same slot +# note2: you can only merge up to 255 updates per accumulator update + + +# this function takes in a list of update_data and: +# - for each of the update data, check if the vaa is the same +# - if the vaa is the same, we combine them, the way we combine them is: +# - we append the updates and increment num_updates +# - if its not the same, we dont combine it +def compress_update_data_list(update_data_list, encoding) -> List[str]: + parsed_data_dict = {} # use a dictionary for O(1) lookup + # combine the ones with the same vaa to a list + for update_data in update_data_list: + parsed_update_data = parse_update_data(update_data, encoding) + vaa = parsed_update_data["vaa"] + + if vaa not in parsed_data_dict: + parsed_data_dict[vaa] = [] + parsed_data_dict[vaa].append(parsed_update_data) + parsed_data_list = list(parsed_data_dict.values()) + + # check if the number of updates in each combined_data is less than 255 + # if any of them are more than 255, we need to chunk them into size of 255 each + + # combine the ones with the same vaa to { same_vaa, num_updates, [updates1, updates2, ...] } + combined_data_list = [] + for parsed_data in parsed_data_list: + combined_data = { + "magic": parsed_data[0]["magic"], + "major_version": parsed_data[0]["major_version"], + "minor_version": parsed_data[0]["minor_version"], + "trailing_header_size": parsed_data[0]["trailing_header_size"], + "update_type": parsed_data[0]["update_type"], + "vaa_length": parsed_data[0]["vaa_length"], + "vaa": parsed_data[0]["vaa"], + "num_updates": sum(len(item["updates"]) for item in parsed_data), + "updates": [update for item in parsed_data for update in item["updates"]], + } + combined_data_list.append(combined_data) + + # serialize it into the accumulator update format + serialized_data_list = [] + for combined_data in combined_data_list: + serialized_data = serialize_combined_data(combined_data) + serialized_data_list.append(serialized_data) + + return serialized_data_list + + +def serialize_combined_data(combined_data): + serialized_data = bytearray() + serialized_data.extend(combined_data["magic"]) + serialized_data.append(combined_data["major_version"]) + serialized_data.append(combined_data["minor_version"]) + serialized_data.append(combined_data["trailing_header_size"]) + serialized_data.append(combined_data["update_type"]) + serialized_data.extend(combined_data["vaa_length"].to_bytes(2, byteorder="big")) + serialized_data.extend(combined_data["vaa"]) + serialized_data.append(combined_data["num_updates"]) + for update in combined_data["updates"]: + serialized_data.extend(update["message_size"].to_bytes(2, byteorder="big")) + serialized_data.extend(update["message"]) + serialized_data.append(update["proof_size"]) + for proof in update["proof"]: + serialized_data.extend(proof) + return serialized_data.hex() + + +def parse_update_data(update_data, encoding): + encoded_update_data = encode_vaa_for_chain(update_data, encoding, buffer=True) + offset = 0 + magic = encoded_update_data[offset : offset + 4] + offset += 4 + major_version = encoded_update_data[offset] + offset += 1 + minor_version = encoded_update_data[offset] + offset += 1 + + trailing_header_size = encoded_update_data[offset] + offset += 1 + trailing_header_size + + update_type = encoded_update_data[offset] + offset += 1 + + if update_type != 0: + logger.info(f"Invalid accumulator update type: {update_type}") + return None + + vaa_length = int.from_bytes( + encoded_update_data[offset : offset + 2], byteorder="big" + ) + offset += 2 + + vaa_buffer = encoded_update_data[offset : offset + vaa_length] + offset += vaa_length + + num_updates = encoded_update_data[offset] + offset += 1 + + updates = [] + for _ in range(num_updates): + message_size = int.from_bytes( + encoded_update_data[offset : offset + 2], byteorder="big" + ) + offset += 2 + + message = encoded_update_data[offset : offset + message_size] + offset += message_size + + proof_size = encoded_update_data[offset] + offset += 1 + + proof = [] + for _ in range(proof_size): + hash = encoded_update_data[offset : offset + 20] + proof.append(hash) + offset += 20 + + updates.append( + { + "message_size": message_size, + "message": message, + "proof_size": proof_size, + "proof": proof, + } + ) + return { + "magic": magic, + "major_version": major_version, + "minor_version": minor_version, + "trailing_header_size": trailing_header_size, + "update_type": update_type, + "vaa_length": vaa_length, + "vaa": vaa_buffer, + "num_updates": num_updates, + "updates": updates, + } From 94e0402b669e7f8022c5c0ec266d5b235830859f Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 4 Sep 2023 17:23:01 +0900 Subject: [PATCH 2/7] add comments and refactor --- pythclient/price_feeds.py | 271 ++++++++++++++++++++++++++++---------- 1 file changed, 201 insertions(+), 70 deletions(-) diff --git a/pythclient/price_feeds.py b/pythclient/price_feeds.py index f83acbf..911cae8 100644 --- a/pythclient/price_feeds.py +++ b/pythclient/price_feeds.py @@ -115,6 +115,54 @@ def to_dict(self, verbose=False, vaa_format=DEFAULT_VAA_ENCODING): return result +class MerkleUpdate: + def __init__( + self, message_size: int, message: bytes, proof_size: int, proof: List[bytes] + ): + self.message_size = message_size + self.message = message + self.proof_size = proof_size + self.proof = proof + + def __str__(self): + return ( + f"MerkleUpdate(message_size={self.message_size}, message={self.message}, " + f"proof_size={self.proof_size}, proof={self.proof})" + ) + + +class AccumulatorUpdate: + def __init__( + self, + magic: bytes, + major_version: int, + minor_version: int, + trailing_header_size: int, + update_type: int, + vaa_length: int, + vaa: bytes, + num_updates: int, + updates: List[MerkleUpdate], + ): + self.magic = magic + self.major_version = major_version + self.minor_version = minor_version + self.trailing_header_size = trailing_header_size + self.update_type = update_type + self.vaa_length = vaa_length + self.vaa = vaa + self.num_updates = num_updates + self.updates = updates + + def __str__(self): + return ( + f"AccumulatorUpdate(magic={self.magic}, major_version={self.major_version}, " + f"minor_version={self.minor_version}, trailing_header_size={self.trailing_header_size}, " + f"update_type={self.update_type}, vaa_length={self.vaa_length}, vaa={self.vaa}, " + f"num_updates={self.num_updates}, updates={self.updates})" + ) + + # Referenced from https://github.com/pyth-network/pyth-crosschain/blob/110caed6be3be7885773d2f6070b143cc13fb0ee/price_service/server/src/encoding.ts#L24 def encode_vaa_for_chain(vaa, vaa_format, buffer=False): # check if vaa is already in vaa_format @@ -344,6 +392,8 @@ def parse_price_attestation(bytes_): # Referenced from https://github.com/pyth-network/pyth-crosschain/blob/110caed6be3be7885773d2f6070b143cc13fb0ee/price_service/server/src/rest.ts#L139 def vaa_to_price_infos(vaa, encoding=DEFAULT_VAA_ENCODING) -> List[PriceInfo]: + if encode_vaa_for_chain(vaa, encoding, buffer=True)[:4].hex() == ACCUMULATOR_MAGIC: + return extract_price_info_from_accumulator_update(vaa, encoding) parsed_vaa = parse_vaa(vaa, encoding) batch_attestation = parse_batch_price_attestation(parsed_vaa["payload"]) price_infos = [] @@ -356,18 +406,39 @@ def vaa_to_price_infos(vaa, encoding=DEFAULT_VAA_ENCODING) -> List[PriceInfo]: parsed_vaa["emitter_chain"], ) ) - return price_infos -def vaa_to_price_info(price_feed_id, vaa, encoding=DEFAULT_VAA_ENCODING) -> PriceInfo: - if encode_vaa_for_chain(vaa, encoding, buffer=True)[:4].hex() == ACCUMULATOR_MAGIC: - return extract_price_info_from_accumulator_update(price_feed_id, vaa, encoding) +def vaa_to_price_info(id, vaa, encoding=DEFAULT_VAA_ENCODING) -> Optional[PriceInfo]: + """ + This function retrieves a specific PriceInfo object from a given VAA. + + Parameters: + id (str): The id of the PriceInfo object to find. + vaa (str): The VAA from which to generate the PriceInfo object. + encoding (str, optional): The encoding of the VAA. Defaults to hex. + + Returns: + Optional[PriceInfo]: The PriceInfo object with the given id. If no such object is found, returns None. + """ price_infos = vaa_to_price_infos(vaa, encoding) + return find_price_info_by_id(price_infos, id) + + +def find_price_info_by_id(price_infos, id): + """ + This function searches through a list of PriceInfo objects and returns the first one that matches the provided id. + + Parameters: + price_infos (List[PriceInfo]): A list of PriceInfo objects to search through. + id (str): The id of the PriceInfo object to find. + + Returns: + PriceInfo: The first PriceInfo object in the list that matches the provided id. If no match is found, returns None. + """ for price_info in price_infos: - if price_info.price_feed.id == price_feed_id: + if price_info.price_feed.id == id: return price_info - return None @@ -539,24 +610,23 @@ def extract_price_info_from_accumulator_update( return price_infos -# 1. extract a list of [slot, wh_proof, price_update_message, merkle_proof] per updateData -# 2. combine the ones with the same slot to { same_slot, wh_proof, [price_update_message, merkle_proof] } -# 3. serialize it into the accumulator update format - -# note: you can only merge those with the same slot -# note2: you can only merge up to 255 updates per accumulator update +def compress_accumulator_update(update_data_list, encoding) -> List[str]: + """ + This function compresses a list of accumulator update data by combining those with the same VAA. + It also splits the combined updates into chunks of 255 updates each, as per the limit. + Parameters: + update_data_list (List[str]): A list of accumulator update data to compress. + encoding (str): The encoding of the update data. -# this function takes in a list of update_data and: -# - for each of the update data, check if the vaa is the same -# - if the vaa is the same, we combine them, the way we combine them is: -# - we append the updates and increment num_updates -# - if its not the same, we dont combine it -def compress_update_data_list(update_data_list, encoding) -> List[str]: - parsed_data_dict = {} # use a dictionary for O(1) lookup - # combine the ones with the same vaa to a list + Returns: + List[str]: A list of serialized accumulator update data. Each item in the list is a hexadecimal string representing + an accumulator update data. The updates with the same VAA are combined and split into chunks of 255 updates each. + """ + parsed_data_dict = {} # Use a dictionary for O(1) lookup + # Combine the ones with the same VAA to a list for update_data in update_data_list: - parsed_update_data = parse_update_data(update_data, encoding) + parsed_update_data = parse_accumulator_update(update_data, encoding) vaa = parsed_update_data["vaa"] if vaa not in parsed_data_dict: @@ -564,54 +634,115 @@ def compress_update_data_list(update_data_list, encoding) -> List[str]: parsed_data_dict[vaa].append(parsed_update_data) parsed_data_list = list(parsed_data_dict.values()) - # check if the number of updates in each combined_data is less than 255 - # if any of them are more than 255, we need to chunk them into size of 255 each - - # combine the ones with the same vaa to { same_vaa, num_updates, [updates1, updates2, ...] } + # Combines accumulator update data with the same VAA into a single dictionary combined_data_list = [] for parsed_data in parsed_data_list: - combined_data = { - "magic": parsed_data[0]["magic"], - "major_version": parsed_data[0]["major_version"], - "minor_version": parsed_data[0]["minor_version"], - "trailing_header_size": parsed_data[0]["trailing_header_size"], - "update_type": parsed_data[0]["update_type"], - "vaa_length": parsed_data[0]["vaa_length"], - "vaa": parsed_data[0]["vaa"], - "num_updates": sum(len(item["updates"]) for item in parsed_data), - "updates": [update for item in parsed_data for update in item["updates"]], - } + combined_data = AccumulatorUpdate( + magic=parsed_data[0]["magic"], + major_version=parsed_data[0]["major_version"], + minor_version=parsed_data[0]["minor_version"], + trailing_header_size=parsed_data[0]["trailing_header_size"], + update_type=parsed_data[0]["update_type"], + vaa_length=parsed_data[0]["vaa_length"], + vaa=parsed_data[0]["vaa"], + num_updates=sum(len(item["updates"]) for item in parsed_data), + updates=[ + update for item in parsed_data for update in item["updates"] + ], # Flatten the list of all 'updates' lists in the parsed data + ) combined_data_list.append(combined_data) - # serialize it into the accumulator update format - serialized_data_list = [] + # Chunk the combined updates into chunks of 255 updates each + chunked_updates = [] for combined_data in combined_data_list: - serialized_data = serialize_combined_data(combined_data) + chunked_updates.extend( + [ + combined_data["updates"][i : i + MAX_MESSAGE_IN_SINGLE_UPDATE_DATA] + for i in range( + 0, len(combined_data["updates"]), MAX_MESSAGE_IN_SINGLE_UPDATE_DATA + ) + ] + ) + + # Serialize it into the accumulator update format + serialized_data_list = [] + for update in chunked_updates: + serialized_data = serialize_accumulator_update(update, encoding) serialized_data_list.append(serialized_data) return serialized_data_list -def serialize_combined_data(combined_data): +def serialize_accumulator_update(data, encoding): + """ + This function serializes an accumulator update data into a hexadecimal string. + + Parameters: + data (dict): The accumulator update data to serialize. The dictionary should include the following keys: + - "magic": The magic of the update data. + - "major_version": The major version of the update data. + - "minor_version": The minor version of the update data. + - "trailing_header_size": The size of the trailing header in the update data. + - "update_type": The type of the update. + - "vaa_length": The length of the VAA. + - "vaa": The VAA itself. + - "num_updates": The number of updates in the update data. + - "updates": A list of dictionaries, where each dictionary represents an update and includes the following keys: + - "message_size": The size of the message in the update. + - "message": The message itself. + - "proof_size": The size of the proof in the update. + - "proof": The proof itself. + + Returns: + str: The serialized accumulator update data as a hexadecimal string. + """ serialized_data = bytearray() - serialized_data.extend(combined_data["magic"]) - serialized_data.append(combined_data["major_version"]) - serialized_data.append(combined_data["minor_version"]) - serialized_data.append(combined_data["trailing_header_size"]) - serialized_data.append(combined_data["update_type"]) - serialized_data.extend(combined_data["vaa_length"].to_bytes(2, byteorder="big")) - serialized_data.extend(combined_data["vaa"]) - serialized_data.append(combined_data["num_updates"]) - for update in combined_data["updates"]: + serialized_data.extend(data["magic"]) + serialized_data.append(data["major_version"]) + serialized_data.append(data["minor_version"]) + serialized_data.append(data["trailing_header_size"]) + serialized_data.append(data["update_type"]) + serialized_data.extend(data["vaa_length"].to_bytes(2, byteorder="big")) + serialized_data.extend(data["vaa"]) + serialized_data.append(data["num_updates"]) + for update in data["updates"]: serialized_data.extend(update["message_size"].to_bytes(2, byteorder="big")) serialized_data.extend(update["message"]) serialized_data.append(update["proof_size"]) for proof in update["proof"]: serialized_data.extend(proof) - return serialized_data.hex() - - -def parse_update_data(update_data, encoding): + if encoding == "hex": + return serialized_data.hex() + else: + return base64.b64encode(serialized_data).decode("ascii") + + +def parse_accumulator_update(update_data, encoding): + """ + This function parses an accumulator update data. + + Parameters: + update_data (str): The accumulator update data to parse. + encoding (str): The encoding of the update data. + + Returns: + AccumulatorUpdate: An AccumulatorUpdate object containing the parsed accumulator update data. The object includes the following attributes: + - "magic": The magic of the update data. + - "major_version": The major version of the update data. + - "minor_version": The minor version of the update data. + - "trailing_header_size": The size of the trailing header in the update data. + - "update_type": The type of the update. + - "vaa_length": The length of the VAA. + - "vaa": The VAA itself. + - "num_updates": The number of updates in the update data. + - "updates": A list of MerkleUpdate objects, where each update includes the following attributes: + - "message_size": The size of the message in the update. + - "message": The message itself. + - "proof_size": The size of the proof in the update. + - "proof": The proof itself. + + If the update type is not 0, the function logs an info message and returns None. + """ encoded_update_data = encode_vaa_for_chain(update_data, encoding, buffer=True) offset = 0 magic = encoded_update_data[offset : offset + 4] @@ -662,21 +793,21 @@ def parse_update_data(update_data, encoding): offset += 20 updates.append( - { - "message_size": message_size, - "message": message, - "proof_size": proof_size, - "proof": proof, - } + MerkleUpdate( + message_size=message_size, + message=message, + proof_size=proof_size, + proof=proof, + ) ) - return { - "magic": magic, - "major_version": major_version, - "minor_version": minor_version, - "trailing_header_size": trailing_header_size, - "update_type": update_type, - "vaa_length": vaa_length, - "vaa": vaa_buffer, - "num_updates": num_updates, - "updates": updates, - } + return AccumulatorUpdate( + magic=magic, + major_version=major_version, + minor_version=minor_version, + trailing_header_size=trailing_header_size, + update_type=update_type, + vaa_length=vaa_length, + vaa=vaa_buffer, + num_updates=num_updates, + updates=updates, + ) From 4beaf8c18a3bb9f2755bf336b41de863ef858a48 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 4 Sep 2023 23:35:33 +0900 Subject: [PATCH 3/7] more refactor --- pythclient/price_feeds.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pythclient/price_feeds.py b/pythclient/price_feeds.py index 911cae8..f050c72 100644 --- a/pythclient/price_feeds.py +++ b/pythclient/price_feeds.py @@ -675,7 +675,7 @@ def compress_accumulator_update(update_data_list, encoding) -> List[str]: def serialize_accumulator_update(data, encoding): """ - This function serializes an accumulator update data into a hexadecimal string. + This function serializes an accumulator update data into a string. Parameters: data (dict): The accumulator update data to serialize. The dictionary should include the following keys: @@ -692,9 +692,10 @@ def serialize_accumulator_update(data, encoding): - "message": The message itself. - "proof_size": The size of the proof in the update. - "proof": The proof itself. + encoding (str): The encoding of the serialized data. Can be either "hex" or "base64". Returns: - str: The serialized accumulator update data as a hexadecimal string. + str: The serialized accumulator update data as a string. If the encoding is "hex", the function returns a hexadecimal string. If the encoding is "base64", the function returns a base64 string. """ serialized_data = bytearray() serialized_data.extend(data["magic"]) From 064b613e7ac9225c9562e20981c999d0d9830023 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 5 Sep 2023 15:14:14 +0900 Subject: [PATCH 4/7] add test --- pythclient/price_feeds.py | 72 +++++----- tests/test_price_feeds.py | 275 +++++++++++++++++++++++++++++++++++--- 2 files changed, 294 insertions(+), 53 deletions(-) diff --git a/pythclient/price_feeds.py b/pythclient/price_feeds.py index f050c72..c465d02 100644 --- a/pythclient/price_feeds.py +++ b/pythclient/price_feeds.py @@ -584,7 +584,7 @@ def extract_price_info_from_accumulator_update( price_infos.append( PriceInfo( seq_num=parsed_vaa["sequence"], - vaa=update_data, + vaa=vaa_str, publish_time=publish_time, attestation_time=publish_time, last_attested_publish_time=prev_publish_time, @@ -627,7 +627,7 @@ def compress_accumulator_update(update_data_list, encoding) -> List[str]: # Combine the ones with the same VAA to a list for update_data in update_data_list: parsed_update_data = parse_accumulator_update(update_data, encoding) - vaa = parsed_update_data["vaa"] + vaa = parsed_update_data.vaa if vaa not in parsed_data_dict: parsed_data_dict[vaa] = [] @@ -638,16 +638,16 @@ def compress_accumulator_update(update_data_list, encoding) -> List[str]: combined_data_list = [] for parsed_data in parsed_data_list: combined_data = AccumulatorUpdate( - magic=parsed_data[0]["magic"], - major_version=parsed_data[0]["major_version"], - minor_version=parsed_data[0]["minor_version"], - trailing_header_size=parsed_data[0]["trailing_header_size"], - update_type=parsed_data[0]["update_type"], - vaa_length=parsed_data[0]["vaa_length"], - vaa=parsed_data[0]["vaa"], - num_updates=sum(len(item["updates"]) for item in parsed_data), + magic=parsed_data[0].magic, + major_version=parsed_data[0].major_version, + minor_version=parsed_data[0].minor_version, + trailing_header_size=parsed_data[0].trailing_header_size, + update_type=parsed_data[0].update_type, + vaa_length=parsed_data[0].vaa_length, + vaa=parsed_data[0].vaa, + num_updates=sum(len(item.updates) for item in parsed_data), updates=[ - update for item in parsed_data for update in item["updates"] + update for item in parsed_data for update in item.updates ], # Flatten the list of all 'updates' lists in the parsed data ) combined_data_list.append(combined_data) @@ -655,14 +655,22 @@ def compress_accumulator_update(update_data_list, encoding) -> List[str]: # Chunk the combined updates into chunks of 255 updates each chunked_updates = [] for combined_data in combined_data_list: - chunked_updates.extend( - [ - combined_data["updates"][i : i + MAX_MESSAGE_IN_SINGLE_UPDATE_DATA] - for i in range( - 0, len(combined_data["updates"]), MAX_MESSAGE_IN_SINGLE_UPDATE_DATA - ) - ] - ) + for i in range( + 0, len(combined_data.updates), MAX_MESSAGE_IN_SINGLE_UPDATE_DATA + ): + chunk = combined_data.updates[i : i + MAX_MESSAGE_IN_SINGLE_UPDATE_DATA] + chunked_update = AccumulatorUpdate( + magic=combined_data.magic, + major_version=combined_data.major_version, + minor_version=combined_data.minor_version, + trailing_header_size=combined_data.trailing_header_size, + update_type=combined_data.update_type, + vaa_length=combined_data.vaa_length, + vaa=combined_data.vaa, + num_updates=len(chunk), + updates=chunk, + ) + chunked_updates.append(chunked_update) # Serialize it into the accumulator update format serialized_data_list = [] @@ -698,19 +706,19 @@ def serialize_accumulator_update(data, encoding): str: The serialized accumulator update data as a string. If the encoding is "hex", the function returns a hexadecimal string. If the encoding is "base64", the function returns a base64 string. """ serialized_data = bytearray() - serialized_data.extend(data["magic"]) - serialized_data.append(data["major_version"]) - serialized_data.append(data["minor_version"]) - serialized_data.append(data["trailing_header_size"]) - serialized_data.append(data["update_type"]) - serialized_data.extend(data["vaa_length"].to_bytes(2, byteorder="big")) - serialized_data.extend(data["vaa"]) - serialized_data.append(data["num_updates"]) - for update in data["updates"]: - serialized_data.extend(update["message_size"].to_bytes(2, byteorder="big")) - serialized_data.extend(update["message"]) - serialized_data.append(update["proof_size"]) - for proof in update["proof"]: + serialized_data.extend(data.magic) + serialized_data.append(data.major_version) + serialized_data.append(data.minor_version) + serialized_data.append(data.trailing_header_size) + serialized_data.append(data.update_type) + serialized_data.extend(data.vaa_length.to_bytes(2, byteorder="big")) + serialized_data.extend(data.vaa) + serialized_data.append(data.num_updates) + for update in data.updates: + serialized_data.extend(update.message_size.to_bytes(2, byteorder="big")) + serialized_data.extend(update.message) + serialized_data.append(update.proof_size) + for proof in update.proof: serialized_data.extend(proof) if encoding == "hex": return serialized_data.hex() diff --git a/tests/test_price_feeds.py b/tests/test_price_feeds.py index 4e7c627..b2ffb89 100644 --- a/tests/test_price_feeds.py +++ b/tests/test_price_feeds.py @@ -1,13 +1,42 @@ -from pythclient.price_feeds import encode_vaa_for_chain, vaa_to_price_info +import base64 -ID = "e62df6c8b4a85fe1a67db44dc12de5db330f7ac66b72dc658afedf0f4a415b43" # BTC/USD +from pythclient.price_feeds import ( + AccumulatorUpdate, + MerkleUpdate, + compress_accumulator_update, + encode_vaa_for_chain, + parse_accumulator_update, + serialize_accumulator_update, + vaa_to_price_info, + vaa_to_price_infos, +) + +BTC_ID = "e62df6c8b4a85fe1a67db44dc12de5db330f7ac66b72dc658afedf0f4a415b43" # BTC/USD +ETH_ID = "ff61491a931112ddf1bd8147cd1b641375f79f5825126d665480874634fd0ace" # ETH/USD +SOL_ID = "ef0d8b6fda2ceba41da15d4095d1da392a0d2f8ed0c6c7bc0f4cfac8c280b56d" # SOL/USD HEX_VAA = "01000000030d00ca037dde1419ea373de7474bff80268e41f55e84dd8e4fa5e5167808111e63e4064efcfd93075d9d162090bc478a017603ef56c4d26e0c46e0af6128c08984470002e9abd7c4e62bbc6b453c5ca014b7313bbadb17d6443190d6679b7c0ccb03bae619d21dcb5d6d5a484f9e4fd5586dca7c20403dc80d61dbe682a436b4d15d10bc0003a5e748513b75e34d9db61c6fd0fa55533e8424d88fbe6cda8e7cf61d50cb9467192a5cbd47b14b205b38123ad4b7c39d6ce7d6bf02c4a1452d135dc29fc83ef600049e4cd56e3c2e0cfc4cc831bc4ec54410590c15b4b20b63cc91be3436a4d09c0c22be4ab96962ad6e94d83e14c770e122aebae6fdbdea97f98cec7da5d30ed2a40106a7565c6387a700e02ee09652b40bdeef044441ca1e3b3c9376d4a129d22ca861501c3f5c0e8469c9a0e5d1b09d9f84c6517c0a2b400c0b47552006fff1dad3a5000a4db87004c483f899b5fd766c14334dfb5ca2fa5698964cf9644669b325bd3485207cbc4180a360023d1412da68bb11a0a82fee70a6bf03dda30b7aae53e0e465010ba3a6e45c9d8ef1d1041fdc7a926a9f41075531d45824144bbc720d111ee7270a77dd6dd65558b30d0f03692e075bd7d96cdfb24f5a68fecc22e441ded230c9cc010c09380e394e2b30fd036f13152b115dab7a206270d52255dfbbf0505c67bf510e70d0a6075f9bae19235eaf8a0893a4af9ed0df1b8cd67e1fe7b2ec61178d3ca4010dc491600d07d10a6468fb5955d94bc114efab46104e2ae530931231fea52cf7e32964a1c8bfe0ee38aaa8abfe8edcb7c079b6dd97b2c317c9d71cb5973bb53c72010f787e3c59ac484fdca7d5e41b29cebee08cb1789d61a0f29ccd0353118fd667ab1473a626eb6c237cff70ffb1eb2a556862197b08f183d5852168f5ce0f92632b0110f7ee4abdedc936ebebe86b3493292a9fa6625ab910b4a1340b46478a819508d1261f3d559d5cc95dead635c215b80b1cb2df348639d1ca572d3d14f07dc38908011103e3cdc9936ffbb7c0af5d77a4c092c5c42de161c9254919d19af718defd71a757fcbb1e3772e72c3a8c8291ab36f628a060030abf8ffb43923bb1a05cf9605d0112ddea2ce8ec77b9e222db5f1a95861c3da2ed3f54f7e937008bcc14b2458b98990eeb5910c7e9b2a27ff47a9568d0a3fedc12f357323905cbc8a1be6acbc5986b0064c37bca00000000001af8cd23c2ab91237730770bbea08d61005cdda0984348f3f6eecb559638c0bba0000000002144b1420150325748000300010001020005009d2efa1235ab86c0935cb424b102be4f217e74d1109df9e75dfa8338fc0f0908782f95862b045670cd22bee3114c39763a4a08beeb663b145d283c31d7d1101c4f000000059cc51c400000000000e4e1bffffffff8000000059b3f3c700000000000eae895010000001a0000001e0000000064c37bca0000000064c37bca0000000064c37bc9000000059cc51c400000000000e4e1bf0000000064c37bc948d6033d733e27950c2e0351e2505491cd9154824f716d9513514c74b9f98f583dd2b63686a450ec7290df3a1e0b583c0481f651351edfa7636f39aed55cf8a300000005a7462c060000000000d206a2fffffff800000005a5c499380000000000f44b7d010000001c000000200000000064c37bca0000000064c37bca0000000064c37bc900000005a74653cc0000000000d1dedc0000000064c37bc83515b3861e8fe93e5f540ba4077c216404782b86d5e78077b3cbfd27313ab3bce62df6c8b4a85fe1a67db44dc12de5db330f7ac66b72dc658afedf0f4a415b43000002a724c9d1000000000032396fc5fffffff8000002a6e3e0fec0000000002ee4815c010000001b000000200000000064c37bca0000000064c37bca0000000064c37bc9000002a724c9d1000000000032396fc50000000064c37bc99b5f73e0075e7d70376012180ddba94272f68d85eae4104e335561c982253d41a19d04ac696c7a6616d291c7e5d1377cc8be437c327b75adb5dc1bad745fcae800000000045152eb000000000001bb63fffffff800000000044de0b500000000000185df0100000015000000160000000064c37bca0000000064c37bca0000000064c37bc900000000045152eb000000000001bb630000000064c37bc8e876fcd130add8984a33aab52af36bc1b9f822c9ebe376f3aa72d630974e15f0dcef50dd0a4cd2dcc17e45df1676dcb336a11a61c69df7a0299b0150c672d25c000000000074990500000000000011d9fffffff80000000000748c2f000000000000116f010000001b000000200000000064c37bca0000000064c37bca0000000064c37bc900000000007498d400000000000011a80000000064c37bc9" BASE64_VAA = "AQAAAAMNAMoDfd4UGeo3PedHS/+AJo5B9V6E3Y5PpeUWeAgRHmPkBk78/ZMHXZ0WIJC8R4oBdgPvVsTSbgxG4K9hKMCJhEcAAumr18TmK7xrRTxcoBS3MTu62xfWRDGQ1mebfAzLA7rmGdIdy11tWkhPnk/VWG3KfCBAPcgNYdvmgqQ2tNFdELwAA6XnSFE7deNNnbYcb9D6VVM+hCTYj75s2o589h1Qy5RnGSpcvUexSyBbOBI61LfDnWzn1r8CxKFFLRNdwp/IPvYABJ5M1W48Lgz8TMgxvE7FRBBZDBW0sgtjzJG+NDak0JwMIr5KuWlirW6U2D4Ux3DhIq665v296pf5jOx9pdMO0qQBBqdWXGOHpwDgLuCWUrQL3u8EREHKHjs8k3bUoSnSLKhhUBw/XA6Eacmg5dGwnZ+ExlF8CitADAtHVSAG//Ha06UACk24cATEg/iZtf12bBQzTftcovpWmJZM+WRGabMlvTSFIHy8QYCjYAI9FBLaaLsRoKgv7nCmvwPdowt6rlPg5GUBC6Om5FydjvHRBB/cepJqn0EHVTHUWCQUS7xyDREe5ycKd91t1lVYsw0PA2kuB1vX2Wzfsk9aaP7MIuRB3tIwycwBDAk4DjlOKzD9A28TFSsRXat6IGJw1SJV37vwUFxnv1EOcNCmB1+brhkjXq+KCJOkr57Q3xuM1n4f57LsYReNPKQBDcSRYA0H0QpkaPtZVdlLwRTvq0YQTirlMJMSMf6lLPfjKWShyL/g7jiqqKv+jty3wHm23ZeywxfJ1xy1lzu1PHIBD3h+PFmsSE/cp9XkGynOvuCMsXidYaDynM0DUxGP1merFHOmJutsI3z/cP+x6ypVaGIZewjxg9WFIWj1zg+SYysBEPfuSr3tyTbr6+hrNJMpKp+mYlq5ELShNAtGR4qBlQjRJh89VZ1cyV3q1jXCFbgLHLLfNIY50cpXLT0U8H3DiQgBEQPjzcmTb/u3wK9dd6TAksXELeFhySVJGdGa9xje/XGnV/y7Hjdy5yw6jIKRqzb2KKBgAwq/j/tDkjuxoFz5YF0BEt3qLOjsd7niIttfGpWGHD2i7T9U9+k3AIvMFLJFi5iZDutZEMfpsqJ/9HqVaNCj/twS81cyOQXLyKG+asvFmGsAZMN7ygAAAAAAGvjNI8KrkSN3MHcLvqCNYQBc3aCYQ0jz9u7LVZY4wLugAAAAACFEsUIBUDJXSAADAAEAAQIABQCdLvoSNauGwJNctCSxAr5PIX500RCd+edd+oM4/A8JCHgvlYYrBFZwzSK+4xFMOXY6Sgi+62Y7FF0oPDHX0RAcTwAAAAWcxRxAAAAAAADk4b/////4AAAABZs/PHAAAAAAAOrolQEAAAAaAAAAHgAAAABkw3vKAAAAAGTDe8oAAAAAZMN7yQAAAAWcxRxAAAAAAADk4b8AAAAAZMN7yUjWAz1zPieVDC4DUeJQVJHNkVSCT3FtlRNRTHS5+Y9YPdK2NoakUOxykN86HgtYPASB9lE1Ht+nY285rtVc+KMAAAAFp0YsBgAAAAAA0gai////+AAAAAWlxJk4AAAAAAD0S30BAAAAHAAAACAAAAAAZMN7ygAAAABkw3vKAAAAAGTDe8kAAAAFp0ZTzAAAAAAA0d7cAAAAAGTDe8g1FbOGHo/pPl9UC6QHfCFkBHgrhtXngHezy/0nMTqzvOYt9si0qF/hpn20TcEt5dszD3rGa3LcZYr+3w9KQVtDAAACpyTJ0QAAAAAAMjlvxf////gAAAKm4+D+wAAAAAAu5IFcAQAAABsAAAAgAAAAAGTDe8oAAAAAZMN7ygAAAABkw3vJAAACpyTJ0QAAAAAAMjlvxQAAAABkw3vJm19z4AdefXA3YBIYDdupQnL2jYXq5BBOM1VhyYIlPUGhnQSsaWx6ZhbSkcfl0Td8yL5DfDJ7da213ButdF/K6AAAAAAEUVLrAAAAAAABu2P////4AAAAAARN4LUAAAAAAAGF3wEAAAAVAAAAFgAAAABkw3vKAAAAAGTDe8oAAAAAZMN7yQAAAAAEUVLrAAAAAAABu2MAAAAAZMN7yOh2/NEwrdiYSjOqtSrza8G5+CLJ6+N286py1jCXThXw3O9Q3QpM0tzBfkXfFnbcszahGmHGnfegKZsBUMZy0lwAAAAAAHSZBQAAAAAAABHZ////+AAAAAAAdIwvAAAAAAAAEW8BAAAAGwAAACAAAAAAZMN7ygAAAABkw3vKAAAAAGTDe8kAAAAAAHSY1AAAAAAAABGoAAAAAGTDe8k=" -ACCUMULATOR_UPDATE_DATA = "UE5BVQEAAAADuAEAAAADDQAsKPsmb7Vz7io3taJQKgoi1m/z0kqKgtpmlkv+ZuunX2Iegsf+8fuUtpHPLKgCWPU8PN2x9NyAZz5BY9M3SWwJAALYlM0U7f2GFWfEjKwSJlHZ5sf+n6KXCocVC66ImS2o0TD0SBhTWcp0KdcuzR1rY1jfIHaFpVneroRLbTjNrk/WAAMuAYxPVPf1DR30wYQo12Dbf+in3akTjhKERNQ+nPwRjxAyIQD+52LU3Rh2VL7nOIStMNTiBMaiWHywaPoXowWAAQbillhhX4MR+7h81PfxHIbiXBmER4c5M7spilWKkROb+VXhrqnVJL162t9TdhYk56PDIhvXO1Tm/ldjVJw130y0AAk6qpccfsxDZEmVN8LI4z8739Ni/kb+CB3yW2l2dWhKTjBeNanhK6TCCoNH/jRzWfrjrEk5zjNrUr82JwL4fR1OAQrYZescxbH26m8QHiH+RHzwlXpUKJgbHD5NnWtB7oFb9AFM15jbjd4yIEBEtAlXPE0Q4j+X+DLnCtZbLSQiYNh5AQvz70LTbYry1lEExuUcO+IRJiysw5AFyqZ9Y1E//WKIqgEysfcnHwoOxtDtAc5Z9sTUEYfPqQ1d27k3Yk0X7dvCAQ10cdG0qYHb+bQrYRIKKnb0aeCjkCs0HZQY2fXYmimyfTNfECclmPW9k+CfOvW0JKuFxC1l11zJ3zjsgN/peA8BAQ5oIFQGjq9qmf5gegE1DjuzXsGksKao6nsjTXYIspCczCe2h5KNQ9l5hws11hauUKS20JoOYjHwxPD2x0adJKvkAQ+4UjVcZgVEQP8y3caqUDH81Ikcadz2bESpYg93dpnzZTH6A7Ue+RL34PTNx6cCRzukwQuhiStuyL1WYEIrLI4nABAjGv3EBXjWaPLUj59OzVnGkzxkr6C4KDjMmpsYNzx7I2lp2iQV46TM78El8i9h7twiEDUOSdC5CmfQjRpkP72yABGVAQELUm2/SjkpF0O+/rVDgA/Y2/wMacD1ZDahdyvSNSFThn5NyRYA1JXGgIDxoYeAZgkr1gL1cjCLWiO+Bs9QARIiCvHfIkn2aYhYHQq/u6cHB/2DxE3OgbCZyTv8OVO55hQDkJ1gDwAec+IJ4M5Od4OxWEu+OywhJT7zUmwZko9MAGTeJ+kAAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAAAWllxAUFVV1YAAAAAAAVZ/XAAACcQ8Xfx5wQ+nj1rn6IeTUAy+VER1nUBAFUA5i32yLSoX+GmfbRNwS3l2zMPesZrctxliv7fD0pBW0MAAAKUUTJXzwAAAADDrAZ1////+AAAAABk3ifoAAAAAGTeJ+cAAAKWepR2oAAAAAC/b8SsCasjFzENKvXWwOycuzCVaDWfm0IuuuesmamDKl2lNXss15orlNN+xHVNEEIIq7Xg8GRZGVLt43fkg7xli6EPQ/Nyxl6SixiYteNt1uTTh4M1lQTUjPxKnkE5JEea4RnhOWgmSAWMf8ft4KgE7hvRifV1JP0rOsNgsOYFRbs6iDKW1qLpxgZLMAiOclwS3Tjw2hj8sPfq1NHeVttsBEK5SIM14GjAuD/p2V0+NqHqMHxU/kfftg==" +ACCUMULATOR_UPDATE_SAME_VAA = "AQAAAAMNAFpq7E0Obvl72gKby+ppmEI5Z8mRvVKEspd5WJR8J59qYeYtb1/INY9IwAOKHMFs1c2X+K+4PHLl73+VboKgAdcBAQRPIzGyOezZIEyn8HQnPC4WpFHey1mBSgBAJwT/HtbkMk6N5Ffguk7zZ0A5+wTfX0TZLmO9ssLbA5lwDhLQ5y4BAqSpGBxCk7j9taLP29magR9tAMH4DWP6YRDMp+rSecIqYbcR3EGhNQrweVmxuigi7edrkGzqpzapjJ9cr0uWyzAAA1UBZ+B7yxvzBK5uPNX7H/6ru9/5ZAQlyji0gHXZhQzSFn+MfT003KQh9CPe1lmnwbtq3O+9YppdoaPhOGYMc6EABPyfLBNBRtWjodqDO+YjiN5sTWZEr5P6N2LmbiXsnq0hdg/1TObwjlm8y46E3LqMjvcRkDujt5EkDhz6SGBKcC0BC25XVEhmrUE5Y9tUCsUlJr3gFMf53UW1BMBevcmaHEd8AbBDlSvsmj3U6crs661/8G5Cd78G1O+q/yTV8T0OP6EADBtAjSGE/ALSb9eCypqC4QZFhyophpLX3kHCtApLBT8Wf76M3v9E6Tr9lSvPjn0p5BmNQeuOkvbcT6lYbOtUc8EADZH0nP141ClY9whc8W5wiSsR+LxOatxXeMiMrBWOzcJKfdw8oi6MImVNUwGjjSFqWFu/+sgEErcUjMRPLgBbJToBDp1uhbpKhxmoXkwz7uPesMHeCBVQ2Xb3Bijdfg9KtLypNAtOKmUhhOo5fGpiesg6EK2x/Sz73GHR3yFNvdifBmABD6Blheynt9vUTVPubXNKtTlsGuRNUE5V/+Y0uNZv3DhWUtZZ1RA9EjIT8VQG1KQEXkwO3BtcnvAiyT5huZNaoisBECqLp6ajYNUmPadPDCWf1XiRfhrEvXspJ7WAgxh9rmqaZ5IN2MzPzH3az0zKyk5reHASDVo/yLAvT0et9apsgt8BEdor5LY5cD4WVAXljiZsQA8tmyJij+RH5NkZT9wO5w5aHKjhMDVpqgKteksZDdTZC9IxSlkCQ13eAKN4rNuPv0IBEs6lIQpssitiLPQb6KLnLa9UxyMEJGz4arZwb9EFGVehS3WG6sFqRRPU9W4jcvcNGVsBVhuzmkvv3u9MDJuIzUwBZPEpgQAAAAAAGuEB+u2sWFHjK5sjtflBGowrrEquPtTde4Ed0acupKpxAAAAAACKOWQBQVVXVgAAAAAABYq9dAAAJxDIoeJ4YgtzgZ2ikt9+LNjv29G4yg==" +ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA = "UE5BVQEAAAADuAEAAAADDQBaauxNDm75e9oCm8vqaZhCOWfJkb1ShLKXeViUfCefamHmLW9fyDWPSMADihzBbNXNl/ivuDxy5e9/lW6CoAHXAQEETyMxsjns2SBMp/B0JzwuFqRR3stZgUoAQCcE/x7W5DJOjeRX4LpO82dAOfsE319E2S5jvbLC2wOZcA4S0OcuAQKkqRgcQpO4/bWiz9vZmoEfbQDB+A1j+mEQzKfq0nnCKmG3EdxBoTUK8HlZsbooIu3na5Bs6qc2qYyfXK9LlsswAANVAWfge8sb8wSubjzV+x/+q7vf+WQEJco4tIB12YUM0hZ/jH09NNykIfQj3tZZp8G7atzvvWKaXaGj4ThmDHOhAAT8nywTQUbVo6HagzvmI4jebE1mRK+T+jdi5m4l7J6tIXYP9Uzm8I5ZvMuOhNy6jI73EZA7o7eRJA4c+khgSnAtAQtuV1RIZq1BOWPbVArFJSa94BTH+d1FtQTAXr3JmhxHfAGwQ5Ur7Jo91OnK7Outf/BuQne/BtTvqv8k1fE9Dj+hAAwbQI0hhPwC0m/XgsqaguEGRYcqKYaS195BwrQKSwU/Fn++jN7/ROk6/ZUrz459KeQZjUHrjpL23E+pWGzrVHPBAA2R9Jz9eNQpWPcIXPFucIkrEfi8TmrcV3jIjKwVjs3CSn3cPKIujCJlTVMBo40halhbv/rIBBK3FIzETy4AWyU6AQ6dboW6SocZqF5MM+7j3rDB3ggVUNl29wYo3X4PSrS8qTQLTiplIYTqOXxqYnrIOhCtsf0s+9xh0d8hTb3YnwZgAQ+gZYXsp7fb1E1T7m1zSrU5bBrkTVBOVf/mNLjWb9w4VlLWWdUQPRIyE/FUBtSkBF5MDtwbXJ7wIsk+YbmTWqIrARAqi6emo2DVJj2nTwwln9V4kX4axL17KSe1gIMYfa5qmmeSDdjMz8x92s9MyspOa3hwEg1aP8iwL09HrfWqbILfARHaK+S2OXA+FlQF5Y4mbEAPLZsiYo/kR+TZGU/cDucOWhyo4TA1aaoCrXpLGQ3U2QvSMUpZAkNd3gCjeKzbj79CARLOpSEKbLIrYiz0G+ii5y2vVMcjBCRs+Gq2cG/RBRlXoUt1hurBakUT1PVuI3L3DRlbAVYbs5pL797vTAybiM1MAWTxKYEAAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAAAijlkAUFVV1YAAAAAAAWKvXQAACcQyKHieGILc4GdopLffizY79vRuMoBAFUA5i32yLSoX+GmfbRNwS3l2zMPesZrctxliv7fD0pBW0MAAAJb3eCW3wAAAAA8JOwR////+AAAAABk8SmAAAAAAGTxKYAAAAJdBwk84AAAAAA9yJiiCbEdXozpxwFJH1HPYYuRd8HhJRDnW9MncEdkxFG+r+dTKQQxM9w8mXLRd/UWCyaHZYHtjTTArEV3OaVMoedGhGHHMyXwx/nyPWRdsFJkxbMwp5vP1Kg0XG4o5lxmJgsbtFRfX8pfNTgClepaPpyD53afOdLk/yN6Gd6fiq01xIhclaBw4kFKPrK2AuHy3Tjw2hj8sPfq1NHeVttsBEK5SIO7rH9+iA9g386g9frxjnyxLscgUw==" +ACCUMULATOR_UPDATE_BTC_MESSAGE = "AOYt9si0qF/hpn20TcEt5dszD3rGa3LcZYr+3w9KQVtDAAACW93glt8AAAAAPCTsEf////gAAAAAZPEpgAAAAABk8SmAAAACXQcJPOAAAAAAPciYog==" +ACCUMULATOR_UPDATE_BTC_PROOF = [ + "sR1ejOnHAUkfUc9hi5F3weElEOc=", + "W9MncEdkxFG+r+dTKQQxM9w8mXI=", + "0Xf1Fgsmh2WB7Y00wKxFdzmlTKE=", + "50aEYcczJfDH+fI9ZF2wUmTFszA=", + "p5vP1Kg0XG4o5lxmJgsbtFRfX8o=", + "XzU4ApXqWj6cg+d2nznS5P8jehk=", + "3p+KrTXEiFyVoHDiQUo+srYC4fI=", + "3Tjw2hj8sPfq1NHeVttsBEK5SIM=", + "u6x/fogPYN/OoPX68Y58sS7HIFM=", +] +ACCUMULATOR_UPDATE_DATA_ETH_SAME_VAA = "UE5BVQEAAAADuAEAAAADDQBaauxNDm75e9oCm8vqaZhCOWfJkb1ShLKXeViUfCefamHmLW9fyDWPSMADihzBbNXNl/ivuDxy5e9/lW6CoAHXAQEETyMxsjns2SBMp/B0JzwuFqRR3stZgUoAQCcE/x7W5DJOjeRX4LpO82dAOfsE319E2S5jvbLC2wOZcA4S0OcuAQKkqRgcQpO4/bWiz9vZmoEfbQDB+A1j+mEQzKfq0nnCKmG3EdxBoTUK8HlZsbooIu3na5Bs6qc2qYyfXK9LlsswAANVAWfge8sb8wSubjzV+x/+q7vf+WQEJco4tIB12YUM0hZ/jH09NNykIfQj3tZZp8G7atzvvWKaXaGj4ThmDHOhAAT8nywTQUbVo6HagzvmI4jebE1mRK+T+jdi5m4l7J6tIXYP9Uzm8I5ZvMuOhNy6jI73EZA7o7eRJA4c+khgSnAtAQtuV1RIZq1BOWPbVArFJSa94BTH+d1FtQTAXr3JmhxHfAGwQ5Ur7Jo91OnK7Outf/BuQne/BtTvqv8k1fE9Dj+hAAwbQI0hhPwC0m/XgsqaguEGRYcqKYaS195BwrQKSwU/Fn++jN7/ROk6/ZUrz459KeQZjUHrjpL23E+pWGzrVHPBAA2R9Jz9eNQpWPcIXPFucIkrEfi8TmrcV3jIjKwVjs3CSn3cPKIujCJlTVMBo40halhbv/rIBBK3FIzETy4AWyU6AQ6dboW6SocZqF5MM+7j3rDB3ggVUNl29wYo3X4PSrS8qTQLTiplIYTqOXxqYnrIOhCtsf0s+9xh0d8hTb3YnwZgAQ+gZYXsp7fb1E1T7m1zSrU5bBrkTVBOVf/mNLjWb9w4VlLWWdUQPRIyE/FUBtSkBF5MDtwbXJ7wIsk+YbmTWqIrARAqi6emo2DVJj2nTwwln9V4kX4axL17KSe1gIMYfa5qmmeSDdjMz8x92s9MyspOa3hwEg1aP8iwL09HrfWqbILfARHaK+S2OXA+FlQF5Y4mbEAPLZsiYo/kR+TZGU/cDucOWhyo4TA1aaoCrXpLGQ3U2QvSMUpZAkNd3gCjeKzbj79CARLOpSEKbLIrYiz0G+ii5y2vVMcjBCRs+Gq2cG/RBRlXoUt1hurBakUT1PVuI3L3DRlbAVYbs5pL797vTAybiM1MAWTxKYEAAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAAAijlkAUFVV1YAAAAAAAWKvXQAACcQyKHieGILc4GdopLffizY79vRuMoBAFUA/2FJGpMREt3xvYFHzRtkE3X3n1glEm1mVICHRjT9Cs4AAAAmUE1Q3AAAAAAFW9d5////+AAAAABk8SmAAAAAAGTxKX8AAAAmVojMOAAAAAAFhbrjCf0FL/Gh8P02OPw0rSRrwrAE7A5nqKEYAXfPMLLAvruxrf6PeYXQUdIFoB4lBNnwwG5+fLDPJBFgmMogKsX2reLo9aEuwAaxbUa+HwIouU2VjNCBJX5Kvy+9aUPnDPtrCDVCDN4wkHiRIOOTArEqEzDwXI9xwPWog2gG2pbBnq7ut8UB/5xxi/Kgg1Cb3Tjw2hj8sPfq1NHeVttsBEK5SIO7rH9+iA9g386g9frxjnyxLscgUw==" +ACCUMULATOR_UPDATE_DIFFERENT_VAA = "AQAAAAMNAEoUCbmBnFBnz08OS1e8Id7uQPShgJTRh8QR4ywcnghBbwDmDSp7TARIgYpwRwDo4+JzXYMTW12npgBGC+IikUYBAf7Uub70p3/2srHQTFDwC22vb5yXHgie/l8JRICYtKh5fzk7OzxiwAJ56chOMtfmW4pI1aCbbVa70BqO+BRIrIMAAnvoDBeZbaknaBrZmmJARECvq/rpUzkyu+OnlSJ7CZJtW//uR/fIzYBGvpamlsG+fBV2mc38l1AVGiYCX2ZlL9IAA08KRaKnOiz6KhYTlnOT7E7XSSZHuws9a9qUY6Oo8yMmV3x3F83rpBWpi9Ugo11CDJ9woosSd/DU8pjWXKM+5iMBBIrblxVY0LxqTJxEqGNrCg/RtoX3G1stadhIAUGCImOeUd2VcEQdislbuYHVvoAuOCUyQRdysJmxFQulaBC0xvIBCzXee/DnTZwxUVhcHovzZHk1xjIfherjBbVFr7HzoH5kJGDK+4f4As/ePX+IDw5sHjd6SLnmHzckg1TI+LJJOxsBDDWviC0qacCYq0rgZ7x76InITHDDmG4rjyeqVZBkEExzEi7RpBkNk7ftixprqNsfLpiC8yfWIfOHyt4fa99L+XIBDUdUI9LOtO06MXdkXwEmoZZ502F2d9/K9EjSD0RapBG9eg/DLvD/jWOd8pSvFyuEalHNw262Vs8032Q+2ZQSrmAADhDAWGo0+rNbL+FjaqndEYeNXyL3PHgjRonI6Ls0H7V7CabfI5mTie8GqLMYG3F0UohM1qDQ4eHghs7zM43/q+cAD+oGxkSW3LMPL84meqxfyDI5Sbnf+kWJ5drbeVI4M+L2ZhvBIJONvTdfdstWMqqseLvUXn6qVsKp6vvB9c3NKuABEMiE54SyBxZSZ8p96z+Wf05XiLxcApDOUNoLgFFScA7TIg8lkBMTes6r9Rxd0d9BQmIj8EHOH4o1lOLtCy8M/WYAEVktfgaQ/P3ajp4tCujzLVEU2I/XDXIN/RN3s8G0Tzq6MfBBjdCoIenZD2GJWjHuCjXANYQbofSvM4pB8pML2jIAEt3oVv7ZOHKGIW2t7Jz/SUuZJ+LdkKVmILt5U6+DfbJuY21la6d9i7GiNV0zp7cRN4ehnxfOx61pveIErM4ieYQBZPEpgQAAAAAAGuEB+u2sWFHjK5sjtflBGowrrEquPtTde4Ed0acupKpxAAAAAACKOWUBQVVXVgAAAAAABYq9dQAAJxDce8BPBkmE4xdQ6mzGXeOzdwnZSg==" +ACCUMULATOR_UPDATE_DATA_SOL_DIFFERENT_VAA = "UE5BVQEAAAADuAEAAAADDQBKFAm5gZxQZ89PDktXvCHe7kD0oYCU0YfEEeMsHJ4IQW8A5g0qe0wESIGKcEcA6OPic12DE1tdp6YARgviIpFGAQH+1Lm+9Kd/9rKx0ExQ8Attr2+clx4Inv5fCUSAmLSoeX85Ozs8YsACeenITjLX5luKSNWgm21Wu9AajvgUSKyDAAJ76AwXmW2pJ2ga2ZpiQERAr6v66VM5Mrvjp5UiewmSbVv/7kf3yM2ARr6WppbBvnwVdpnN/JdQFRomAl9mZS/SAANPCkWipzos+ioWE5Zzk+xO10kmR7sLPWvalGOjqPMjJld8dxfN66QVqYvVIKNdQgyfcKKLEnfw1PKY1lyjPuYjAQSK25cVWNC8akycRKhjawoP0baF9xtbLWnYSAFBgiJjnlHdlXBEHYrJW7mB1b6ALjglMkEXcrCZsRULpWgQtMbyAQs13nvw502cMVFYXB6L82R5NcYyH4Xq4wW1Ra+x86B+ZCRgyvuH+ALP3j1/iA8ObB43eki55h83JINUyPiySTsbAQw1r4gtKmnAmKtK4Ge8e+iJyExww5huK48nqlWQZBBMcxIu0aQZDZO37Ysaa6jbHy6YgvMn1iHzh8reH2vfS/lyAQ1HVCPSzrTtOjF3ZF8BJqGWedNhdnffyvRI0g9EWqQRvXoPwy7w/41jnfKUrxcrhGpRzcNutlbPNN9kPtmUEq5gAA4QwFhqNPqzWy/hY2qp3RGHjV8i9zx4I0aJyOi7NB+1ewmm3yOZk4nvBqizGBtxdFKITNag0OHh4IbO8zON/6vnAA/qBsZEltyzDy/OJnqsX8gyOUm53/pFieXa23lSODPi9mYbwSCTjb03X3bLVjKqrHi71F5+qlbCqer7wfXNzSrgARDIhOeEsgcWUmfKfes/ln9OV4i8XAKQzlDaC4BRUnAO0yIPJZATE3rOq/UcXdHfQUJiI/BBzh+KNZTi7QsvDP1mABFZLX4GkPz92o6eLQro8y1RFNiP1w1yDf0Td7PBtE86ujHwQY3QqCHp2Q9hiVox7go1wDWEG6H0rzOKQfKTC9oyABLd6Fb+2ThyhiFtreyc/0lLmSfi3ZClZiC7eVOvg32ybmNtZWunfYuxojVdM6e3ETeHoZ8Xzsetab3iBKzOInmEAWTxKYEAAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAAAijllAUFVV1YAAAAAAAWKvXUAACcQ3HvATwZJhOMXUOpsxl3js3cJ2UoBAFUA7w2Lb9os66QdoV1AldHaOSoNL47Qxse8D0z6yMKAtW0AAAAAdZyz4QAAAAAAGzQd////+AAAAABk8SmBAAAAAGTxKYAAAAAAdajBegAAAAAAGDQdCX7POyhhbRE0fZPLbyONUAJ+6eS8CUz89ZmOaX9iCON0AzYTqrcpKZe189sKyhdvV8wZPkdHj8K7sNCWVP77y4K8q0eIHSbEbiAZv3GQxptsAP/yBqn4QmvmKA8tqiNp2ZwyMvTreSVyWhDVPo+acVFaI5WQHw45KwMm3/L7zcEKr8+pzQl8FKfnH1wy3Tjw2hj8sPfq1NHeVttsBEK5SIPSXMgExPoP1vmy7SymxxR8JIM3ug==" def test_valid_hex_vaa_to_price_info(): - price_info = vaa_to_price_info(ID, HEX_VAA) + price_info = vaa_to_price_info(BTC_ID, HEX_VAA) assert price_info.seq_num == 558149954 assert price_info.vaa == HEX_VAA assert price_info.publish_time == 1690532810 @@ -17,7 +46,7 @@ def test_valid_hex_vaa_to_price_info(): assert price_info.price_feed.ema_price.conf == "786727260" assert price_info.price_feed.ema_price.expo == -8 assert price_info.price_feed.ema_price.publish_time == 1690532810 - assert price_info.price_feed.id == ID + assert price_info.price_feed.id == BTC_ID assert price_info.price_feed.price.price == "2916900000000" assert price_info.price_feed.price.conf == "842624965" assert price_info.price_feed.price.expo == -8 @@ -26,7 +55,7 @@ def test_valid_hex_vaa_to_price_info(): def test_valid_base64_vaa_to_price_info(): - price_info = vaa_to_price_info(ID, BASE64_VAA, "base64") + price_info = vaa_to_price_info(BTC_ID, BASE64_VAA, "base64") assert price_info.seq_num == 558149954 assert price_info.vaa == BASE64_VAA assert price_info.publish_time == 1690532810 @@ -36,7 +65,7 @@ def test_valid_base64_vaa_to_price_info(): assert price_info.price_feed.ema_price.conf == "786727260" assert price_info.price_feed.ema_price.expo == -8 assert price_info.price_feed.ema_price.publish_time == 1690532810 - assert price_info.price_feed.id == ID + assert price_info.price_feed.id == BTC_ID assert price_info.price_feed.price.price == "2916900000000" assert price_info.price_feed.price.conf == "842624965" assert price_info.price_feed.price.expo == -8 @@ -46,7 +75,7 @@ def test_valid_base64_vaa_to_price_info(): def test_invalid_vaa_to_price_info(): try: - vaa_to_price_info(ID, HEX_VAA + "10") + vaa_to_price_info(BTC_ID, HEX_VAA + "10") except ValueError as ve: assert ve.args[0] == "Invalid length: 801, expected: 800" @@ -70,19 +99,223 @@ def test_encode_vaa_for_chain(): def test_valid_accumulator_vaa_to_price_info(): - price_info = vaa_to_price_info(ID, ACCUMULATOR_UPDATE_DATA, "base64") - assert price_info.seq_num == 5921137 - assert price_info.vaa == ACCUMULATOR_UPDATE_DATA - assert price_info.publish_time == 1692280808 - assert price_info.attestation_time == 1692280808 - assert price_info.last_attested_publish_time == 1692280807 - assert price_info.price_feed.ema_price.price == "2845324900000" - assert price_info.price_feed.ema_price.conf == "3211773100" + price_info = vaa_to_price_info( + BTC_ID, ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA, "base64" + ) + assert price_info.seq_num == 9058660 + assert price_info.vaa == ACCUMULATOR_UPDATE_SAME_VAA + assert price_info.publish_time == 1693526400 + assert price_info.attestation_time == 1693526400 + assert price_info.last_attested_publish_time == 1693526400 + assert price_info.price_feed.ema_price.price == "2598573260000" + assert price_info.price_feed.ema_price.conf == "1036556450" assert price_info.price_feed.ema_price.expo == -8 - assert price_info.price_feed.ema_price.publish_time == 1692280808 - assert price_info.price_feed.id == ID - assert price_info.price_feed.price.price == "2836040669135" - assert price_info.price_feed.price.conf == "3282830965" + assert price_info.price_feed.ema_price.publish_time == 1693526400 + assert price_info.price_feed.id == BTC_ID + assert price_info.price_feed.price.price == "2593587762911" + assert price_info.price_feed.price.conf == "1009052689" assert price_info.price_feed.price.expo == -8 - assert price_info.price_feed.price.publish_time == 1692280808 + assert price_info.price_feed.price.publish_time == 1693526400 assert price_info.emitter_chain_id == 26 + + +def test_parse_accumulator_update(): + parsed_update_data = parse_accumulator_update( + ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA, "base64" + ) + assert parsed_update_data.magic == b"PNAU" + assert parsed_update_data.major_version == 1 + assert parsed_update_data.minor_version == 0 + assert parsed_update_data.trailing_header_size == 0 + assert parsed_update_data.update_type == 0 + assert parsed_update_data.vaa == base64.b64decode(ACCUMULATOR_UPDATE_SAME_VAA) + assert parsed_update_data.vaa_length == len( + base64.b64decode(ACCUMULATOR_UPDATE_SAME_VAA) + ) + assert parsed_update_data.num_updates == 1 + assert len(parsed_update_data.updates) == 1 + assert parsed_update_data.updates[0].message_size == len( + base64.b64decode(ACCUMULATOR_UPDATE_BTC_MESSAGE) + ) + assert parsed_update_data.updates[0].message == base64.b64decode( + ACCUMULATOR_UPDATE_BTC_MESSAGE + ) + assert parsed_update_data.updates[0].proof_size == len( + [base64.b64decode(pr) for pr in ACCUMULATOR_UPDATE_BTC_PROOF] + ) + assert parsed_update_data.updates[0].proof == [ + base64.b64decode(pr) for pr in ACCUMULATOR_UPDATE_BTC_PROOF + ] + + +def test_serialize_accumulator_update(): + magic = b"PNAU" + major_version = 1 + minor_version = 0 + trailing_header_size = 0 + update_type = 0 + vaa = base64.b64decode(ACCUMULATOR_UPDATE_SAME_VAA) + vaa_length = len(vaa) + num_updates = 1 + message = base64.b64decode(ACCUMULATOR_UPDATE_BTC_MESSAGE) + proof = [base64.b64decode(pr) for pr in ACCUMULATOR_UPDATE_BTC_PROOF] + updates = [ + MerkleUpdate( + message_size=len(message), + message=message, + proof_size=len(proof), + proof=proof, + ) + ] + + update_data = AccumulatorUpdate( + magic=magic, + major_version=major_version, + minor_version=minor_version, + trailing_header_size=trailing_header_size, + update_type=update_type, + vaa_length=vaa_length, + vaa=vaa, + num_updates=num_updates, + updates=updates, + ) + serialized_update_data = serialize_accumulator_update(update_data, "base64") + assert serialized_update_data == ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA + + +def test_compress_accumulator_update(): + test_data = compress_accumulator_update( + [ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA], "base64" + ) + # Test that compressing 2 accumulator updates with the same vaa returns a single compressed update + compressed_update_data = compress_accumulator_update( + [ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA, ACCUMULATOR_UPDATE_DATA_ETH_SAME_VAA], + "base64", + ) + assert len(compressed_update_data) == 1 + + # Test that they are the same as the original + price_infos = vaa_to_price_infos(compressed_update_data[0], "base64") + for price_info in price_infos: + assert price_info.seq_num == 9058660 + assert price_info.vaa == ACCUMULATOR_UPDATE_SAME_VAA + assert price_info.publish_time == 1693526400 + assert price_info.attestation_time == 1693526400 + assert price_info.price_feed.id in [BTC_ID, ETH_ID] + if price_info.price_feed.id == BTC_ID: + assert price_info.last_attested_publish_time == 1693526400 + assert price_info.price_feed.ema_price.price == "2598573260000" + assert price_info.price_feed.ema_price.conf == "1036556450" + assert price_info.price_feed.ema_price.expo == -8 + assert price_info.price_feed.ema_price.publish_time == 1693526400 + assert price_info.price_feed.price.price == "2593587762911" + assert price_info.price_feed.price.conf == "1009052689" + assert price_info.price_feed.price.expo == -8 + assert price_info.price_feed.price.publish_time == 1693526400 + elif price_info.price_feed.id == ETH_ID: + assert price_info.last_attested_publish_time == 1693526399 + assert price_info.price_feed.ema_price.price == "164660563000" + assert price_info.price_feed.ema_price.conf == "92650211" + assert price_info.price_feed.ema_price.expo == -8 + assert price_info.price_feed.ema_price.publish_time == 1693526400 + assert price_info.price_feed.price.price == "164556001500" + assert price_info.price_feed.price.conf == "89905017" + assert price_info.price_feed.price.expo == -8 + assert price_info.price_feed.price.publish_time == 1693526400 + + # Test that compressing 2 accumulator updates with the same vaa in hex encoding returns a single compressed update + compressed_update_data = compress_accumulator_update( + [ + base64.b64decode(ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA).hex(), + base64.b64decode(ACCUMULATOR_UPDATE_DATA_ETH_SAME_VAA).hex(), + ], + "hex", + ) + assert len(compressed_update_data) == 1 + + # Test that they are the same as the original + price_infos = vaa_to_price_infos(compressed_update_data[0], "hex") + for price_info in price_infos: + assert price_info.seq_num == 9058660 + assert price_info.vaa == base64.b64decode(ACCUMULATOR_UPDATE_SAME_VAA).hex() + assert price_info.publish_time == 1693526400 + assert price_info.attestation_time == 1693526400 + assert price_info.price_feed.id in [BTC_ID, ETH_ID] + if price_info.price_feed.id == BTC_ID: + assert price_info.last_attested_publish_time == 1693526400 + assert price_info.price_feed.ema_price.price == "2598573260000" + assert price_info.price_feed.ema_price.conf == "1036556450" + assert price_info.price_feed.ema_price.expo == -8 + assert price_info.price_feed.ema_price.publish_time == 1693526400 + assert price_info.price_feed.price.price == "2593587762911" + assert price_info.price_feed.price.conf == "1009052689" + assert price_info.price_feed.price.expo == -8 + assert price_info.price_feed.price.publish_time == 1693526400 + elif price_info.price_feed.id == ETH_ID: + assert price_info.last_attested_publish_time == 1693526399 + assert price_info.price_feed.ema_price.price == "164660563000" + assert price_info.price_feed.ema_price.conf == "92650211" + assert price_info.price_feed.ema_price.expo == -8 + assert price_info.price_feed.ema_price.publish_time == 1693526400 + assert price_info.price_feed.price.price == "164556001500" + assert price_info.price_feed.price.conf == "89905017" + assert price_info.price_feed.price.expo == -8 + assert price_info.price_feed.price.publish_time == 1693526400 + + # Test that compressing 2 accumulator updates with different vaas returns 2 compressed updates + compressed_update_data = compress_accumulator_update( + [ + ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA, + ACCUMULATOR_UPDATE_DATA_SOL_DIFFERENT_VAA, + ], + "base64", + ) + assert len(compressed_update_data) == 2 + + # Test that they are the same as the original + price_info = vaa_to_price_info(BTC_ID, compressed_update_data[0], "base64") + assert price_info.seq_num == 9058660 + assert price_info.vaa == ACCUMULATOR_UPDATE_SAME_VAA + assert price_info.publish_time == 1693526400 + assert price_info.attestation_time == 1693526400 + assert price_info.last_attested_publish_time == 1693526400 + assert price_info.price_feed.ema_price.price == "2598573260000" + assert price_info.price_feed.ema_price.conf == "1036556450" + assert price_info.price_feed.ema_price.expo == -8 + assert price_info.price_feed.ema_price.publish_time == 1693526400 + assert price_info.price_feed.id == BTC_ID + assert price_info.price_feed.price.price == "2593587762911" + assert price_info.price_feed.price.conf == "1009052689" + assert price_info.price_feed.price.expo == -8 + assert price_info.price_feed.price.publish_time == 1693526400 + assert price_info.emitter_chain_id == 26 + + price_info = vaa_to_price_info(SOL_ID, compressed_update_data[1], "base64") + assert price_info.seq_num == 9058661 + assert price_info.vaa == ACCUMULATOR_UPDATE_DIFFERENT_VAA + assert price_info.publish_time == 1693526401 + assert price_info.attestation_time == 1693526401 + assert price_info.last_attested_publish_time == 1693526400 + assert price_info.price_feed.ema_price.price == "1973993850" + assert price_info.price_feed.ema_price.conf == "1586205" + assert price_info.price_feed.ema_price.expo == -8 + assert price_info.price_feed.ema_price.publish_time == 1693526401 + assert price_info.price_feed.id == SOL_ID + assert price_info.price_feed.price.price == "1973203937" + assert price_info.price_feed.price.conf == "1782813" + assert price_info.price_feed.price.expo == -8 + assert price_info.price_feed.price.publish_time == 1693526401 + assert price_info.emitter_chain_id == 26 + + # Test that compressing 256 accumulator updates with same vaas returns 2 chunk compressed updates of 255 and 1 + compressed_update_data = compress_accumulator_update( + [ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA] * 256, + "base64", + ) + assert len(compressed_update_data) == 2 + parsed_update_data = parse_accumulator_update(compressed_update_data[0], "base64") + assert parsed_update_data.num_updates == 255 + assert len(parsed_update_data.updates) == 255 + parsed_update_data = parse_accumulator_update(compressed_update_data[1], "base64") + assert parsed_update_data.num_updates == 1 + assert len(parsed_update_data.updates) == 1 From 1c87584101ec209126f934d2757a94bdd8b26a57 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 5 Sep 2023 15:19:32 +0900 Subject: [PATCH 5/7] refactor test --- tests/test_price_feeds.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_price_feeds.py b/tests/test_price_feeds.py index b2ffb89..5b989f4 100644 --- a/tests/test_price_feeds.py +++ b/tests/test_price_feeds.py @@ -1,6 +1,7 @@ import base64 from pythclient.price_feeds import ( + ACCUMULATOR_MAGIC, AccumulatorUpdate, MerkleUpdate, compress_accumulator_update, @@ -123,7 +124,7 @@ def test_parse_accumulator_update(): parsed_update_data = parse_accumulator_update( ACCUMULATOR_UPDATE_DATA_BTC_SAME_VAA, "base64" ) - assert parsed_update_data.magic == b"PNAU" + assert parsed_update_data.magic == bytes.fromhex(ACCUMULATOR_MAGIC) assert parsed_update_data.major_version == 1 assert parsed_update_data.minor_version == 0 assert parsed_update_data.trailing_header_size == 0 @@ -149,7 +150,7 @@ def test_parse_accumulator_update(): def test_serialize_accumulator_update(): - magic = b"PNAU" + magic = bytes.fromhex(ACCUMULATOR_MAGIC) major_version = 1 minor_version = 0 trailing_header_size = 0 From c4ec10fddbd016da5aa23e6eee6ea24a9efe9a17 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 5 Sep 2023 18:36:09 +0900 Subject: [PATCH 6/7] refactor --- pythclient/price_feeds.py | 52 +++------------------------------------ 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/pythclient/price_feeds.py b/pythclient/price_feeds.py index c465d02..0fd4a74 100644 --- a/pythclient/price_feeds.py +++ b/pythclient/price_feeds.py @@ -488,60 +488,16 @@ def price_attestation_to_price_feed(price_attestation): def extract_price_info_from_accumulator_update( update_data, encoding ) -> Optional[Dict[str, Any]]: - encoded_update_data = encode_vaa_for_chain(update_data, encoding, buffer=True) - offset = 0 - offset += 4 # magic - offset += 1 # major version - offset += 1 # minor version - - trailing_header_size = encoded_update_data[offset] - offset += 1 + trailing_header_size - - update_type = encoded_update_data[offset] - offset += 1 - - if update_type != 0: - logger.info(f"Invalid accumulator update type: {update_type}") - return None - - vaa_length = int.from_bytes( - encoded_update_data[offset : offset + 2], byteorder="big" - ) - offset += 2 - - vaa_buffer = encoded_update_data[offset : offset + vaa_length] - # convert vaa_buffer to string based on encoding + parsed_update_data = parse_accumulator_update(update_data, encoding) + vaa_buffer = parsed_update_data.vaa if encoding == "hex": vaa_str = vaa_buffer.hex() elif encoding == "base64": vaa_str = base64.b64encode(vaa_buffer).decode("ascii") parsed_vaa = parse_vaa(vaa_str, encoding) - offset += vaa_length - - num_updates = encoded_update_data[offset] - offset += 1 - price_infos = [] - for _ in range(num_updates): - message_length = int.from_bytes( - encoded_update_data[offset : offset + 2], byteorder="big" - ) # message_size: u16 - offset += 2 - - message = encoded_update_data[ - offset : offset + message_length - ] # message: [u8; message_size] - offset += message_length - - proof_length = encoded_update_data[offset] # proof_size: u8 - offset += 1 - - proof = [] - for _ in range(proof_length): - hash = encoded_update_data[offset : offset + 20] - proof.append(hash) - offset += 20 - + for update in parsed_update_data.updates: + message = update.message message_offset = 0 message_type = message[message_offset] message_offset += 1 From 57f1fd203a512b2813073721d498d6133b3d2330 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 6 Sep 2023 09:20:21 +0900 Subject: [PATCH 7/7] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f5897e0..651aae5 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='pythclient', - version='0.1.12', + version='0.1.13', packages=['pythclient'], author='Pyth Developers', author_email='contact@pyth.network',