From 7523aa2d9d7e5e58090d0023889eff8c2c0e918d Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 21 Feb 2024 14:36:00 +0900 Subject: [PATCH 01/60] update aggregation logic to aggregate in the same slot --- program/c/src/oracle/upd_aggregate.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index 9668a073..b87af35c 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -134,8 +134,8 @@ static inline void upd_twap( // update aggregate price static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timestamp ) { - // only re-compute aggregate in next slot - if ( slot <= ptr->agg_.pub_slot_ ) { + // only re-compute aggregate in current or future slots + if ( slot < ptr->agg_.pub_slot_ ) { return false; } @@ -152,6 +152,8 @@ static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest // update aggregate details ready for next slot ptr->valid_slot_ = ptr->agg_.pub_slot_;// valid slot-time of agg. price + // only update twap if the slot is in the future + bool update_twap = slot > ptr->agg_.pub_slot_; ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price ptr->timestamp_ = timestamp; @@ -224,7 +226,9 @@ static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest ptr->agg_.price_ = agg_price; ptr->agg_.conf_ = (uint64_t)agg_conf; - upd_twap( ptr, agg_diff ); + if ( update_twap ) { + upd_twap( ptr, agg_diff ); + } return true; } From 9c1ad0cca21ce36359a456d6d6e01e255d6ddabc Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 21 Feb 2024 14:37:33 +0900 Subject: [PATCH 02/60] Add features.h to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1897cb58..0c997c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ target cmake-build-* /venv +# CMake files +features.h + # IntelliJ / CLion configuration .idea *.iml From ce47715eff846ffab4515973af96d6de4b2edbad Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 21 Feb 2024 22:01:06 +0900 Subject: [PATCH 03/60] Refactor test_upd_aggregate.rs: Clean up imports and formatting, update test cases --- program/rust/src/tests/test_upd_aggregate.rs | 36 ++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/program/rust/src/tests/test_upd_aggregate.rs b/program/rust/src/tests/test_upd_aggregate.rs index a2f643fb..12808955 100644 --- a/program/rust/src/tests/test_upd_aggregate.rs +++ b/program/rust/src/tests/test_upd_aggregate.rs @@ -78,6 +78,37 @@ fn test_upd_aggregate() { price_account.is_signer = false; PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); + // test same slot aggregation, aggregate price and conf should be updated but not twap and twac + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.num_ = 1; + price_data.last_slot_ = 1000; + price_data.agg_.pub_slot_ = 1000; + price_data.comp_[0].latest_ = p1; + } + unsafe { + assert!(c_upd_aggregate( + price_account.try_borrow_mut_data().unwrap().as_mut_ptr(), + 1000, + 1, + )); + } + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + + assert_eq!(price_data.agg_.price_, 100); + assert_eq!(price_data.agg_.conf_, 10); + assert_eq!(price_data.twap_.val_, 0); + assert_eq!(price_data.twac_.val_, 0); + assert_eq!(price_data.num_qt_, 1); + assert_eq!(price_data.timestamp_, 1); + assert_eq!(price_data.prev_slot_, 0); + assert_eq!(price_data.prev_price_, 0); + assert_eq!(price_data.prev_conf_, 0); + assert_eq!(price_data.prev_timestamp_, 0); + } + // single publisher { let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); @@ -85,6 +116,11 @@ fn test_upd_aggregate() { price_data.last_slot_ = 1000; price_data.agg_.pub_slot_ = 1000; price_data.comp_[0].latest_ = p1; + price_data.prev_slot_ = 0; + price_data.prev_price_ = 0; + price_data.prev_conf_ = 0; + price_data.prev_timestamp_ = 0; + price_data.agg_.status_ = PC_STATUS_UNKNOWN; } unsafe { From 2a2e2d86271aae24b6e48871cf4822fea9221081 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 13 Mar 2024 15:52:01 +0900 Subject: [PATCH 04/60] add prev_twap_, prev_twac_ and prev_price_cumulative --- program/c/src/oracle/oracle.h | 4 +- program/c/src/oracle/upd_aggregate.h | 9 +- program/rust/Cargo.toml | 1 + program/rust/src/accounts/price.rs | 99 +++++++++++++++----- program/rust/src/processor/upd_price.rs | 31 ++++-- program/rust/src/tests/test_upd_aggregate.rs | 45 ++++++++- 6 files changed, 144 insertions(+), 45 deletions(-) diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index b4e95b53..d887d5b6 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -22,9 +22,9 @@ extern "C" { #define PC_PUBKEY_SIZE_64 (PC_PUBKEY_SIZE/sizeof(uint64_t)) #define PC_MAP_TABLE_SIZE 640 - // Total price component slots available +// Total price component slots available #define PC_NUM_COMP_SOLANA 32 -#define PC_NUM_COMP_PYTHNET 128 +#define PC_NUM_COMP_PYTHNET 127 // PC_NUM_COMP - number of price components in use #ifdef PC_PYTHNET diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index b87af35c..e2ff29ea 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -151,10 +151,8 @@ static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest } // update aggregate details ready for next slot - ptr->valid_slot_ = ptr->agg_.pub_slot_;// valid slot-time of agg. price - // only update twap if the slot is in the future - bool update_twap = slot > ptr->agg_.pub_slot_; - ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price + ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price + ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price ptr->timestamp_ = timestamp; // identify valid quotes @@ -226,9 +224,6 @@ static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest ptr->agg_.price_ = agg_price; ptr->agg_.conf_ = (uint64_t)agg_conf; - if ( update_twap ) { - upd_twap( ptr, agg_diff ); - } return true; } diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 2b88bdab..1c2ff744 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -43,6 +43,7 @@ lazy_static = "1.4.0" # to cargo-build-bpf at that point - we manually capture them at # compile-time and pass on to the child process. [features] +default = ["pythnet"] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] library = [] diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index f69926b5..f0fa23a7 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -35,62 +35,113 @@ mod price_pythnet { }, error::OracleError, }, + std::ops::{ + Deref, + DerefMut, + Index, + IndexMut, + }, }; + #[repr(C)] + #[derive(Copy, Clone)] + pub struct PriceComponentArrayWrapper([PriceComponent; PC_NUM_COMP_PYTHNET as usize]); + + // Implementing Index and IndexMut allows PriceComponentArrayWrapper to use array indexing directly, + // such as price_account.comp_[i], making it behave more like a native array or slice. + impl Index for PriceComponentArrayWrapper { + type Output = PriceComponent; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } + } + impl IndexMut for PriceComponentArrayWrapper { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } + } + + // Implementing Deref and DerefMut allows PriceComponentArrayWrapper to use slice methods directly, + // such as len(), making it behave more like a native array or slice. + impl Deref for PriceComponentArrayWrapper { + type Target = [PriceComponent]; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl DerefMut for PriceComponentArrayWrapper { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } + } + + unsafe impl Pod for PriceComponentArrayWrapper { + } + unsafe impl Zeroable for PriceComponentArrayWrapper { + } + /// Pythnet-only extended price account format. This extension is /// an append-only change that adds extra publisher slots and /// PriceCumulative for TWAP processing. #[repr(C)] #[derive(Copy, Clone, Pod, Zeroable)] pub struct PriceAccountPythnet { - pub header: AccountHeader, + pub header: AccountHeader, /// Type of the price account - pub price_type: u32, + pub price_type: u32, /// Exponent for the published prices - pub exponent: i32, + pub exponent: i32, /// Current number of authorized publishers - pub num_: u32, + pub num_: u32, /// Number of valid quotes for the last aggregation - pub num_qt_: u32, + pub num_qt_: u32, /// Last slot with a succesful aggregation (status : TRADING) - pub last_slot_: u64, + pub last_slot_: u64, /// Second to last slot where aggregation was attempted - pub valid_slot_: u64, + pub valid_slot_: u64, /// Ema for price - pub twap_: PriceEma, + pub twap_: PriceEma, /// Ema for confidence - pub twac_: PriceEma, + pub twac_: PriceEma, /// Last time aggregation was attempted - pub timestamp_: i64, + pub timestamp_: i64, /// Minimum valid publisher quotes for a succesful aggregation - pub min_pub_: u8, - pub message_sent_: u8, + pub min_pub_: u8, + pub message_sent_: u8, /// Configurable max latency in slots between send and receive - pub max_latency_: u8, + pub max_latency_: u8, /// Unused placeholder for alignment - pub unused_2_: i8, - pub unused_3_: i32, + pub unused_2_: i8, + pub unused_3_: i32, /// Corresponding product account - pub product_account: Pubkey, + pub product_account: Pubkey, /// Next price account in the list - pub next_price_account: Pubkey, + pub next_price_account: Pubkey, /// Second to last slot where aggregation was succesful (i.e. status : TRADING) - pub prev_slot_: u64, + pub prev_slot_: u64, /// Aggregate price at prev_slot_ - pub prev_price_: i64, + pub prev_price_: i64, /// Confidence interval at prev_slot_ - pub prev_conf_: u64, + pub prev_conf_: u64, /// Timestamp of prev_slot_ - pub prev_timestamp_: i64, + pub prev_timestamp_: i64, /// Last attempted aggregate results - pub agg_: PriceInfo, + pub agg_: PriceInfo, /// Publishers' price components. NOTE(2023-10-06): On Pythnet, not all /// PC_NUM_COMP_PYTHNET slots are used due to stack size /// issues in the C code. For iterating over price components, /// PC_NUM_COMP must be used. - pub comp_: [PriceComponent; PC_NUM_COMP_PYTHNET as usize], + pub comp_: PriceComponentArrayWrapper, + /// Previous EMA for price and confidence + pub prev_twap_: PriceEma, + pub prev_twac_: PriceEma, + /// Previous TWAP cumulative values + pub prev_price_cumulative: PriceCumulative, /// Cumulative sums of aggregative price and confidence used to compute arithmetic moving averages - pub price_cumulative: PriceCumulative, + pub price_cumulative: PriceCumulative, } impl PriceAccountPythnet { diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 55446b0c..c1879388 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -175,20 +175,33 @@ pub fn upd_price( )?; } + // get number of slots from last update + let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; + // Try to update the aggregate #[allow(unused_variables)] - let mut aggregate_updated = false; + let aggregate_updated = unsafe { + c_upd_aggregate( + price_account.try_borrow_mut_data()?.as_mut_ptr(), + clock.slot, + clock.unix_timestamp, + ) + }; - // NOTE: c_upd_aggregate must use a raw pointer to price - // data. Solana's `.borrow_*` methods require exclusive - // access, i.e. no other borrow can exist for the account. - #[allow(unused_assignments)] - if clock.slot > latest_aggregate_price.pub_slot_ { + if aggregate_updated { + // get price data here + let mut price_data = load_checked::(price_account, cmd_args.header.version)?; + // check if its the first price update in the slot + if slots_since_last_update > 0 { + price_data.prev_twap_ = price_data.twap_; + price_data.prev_twac_ = price_data.twac_; + } + price_data.twap_ = price_data.prev_twap_; + price_data.twac_ = price_data.prev_twac_; unsafe { - aggregate_updated = c_upd_aggregate( + c_upd_twap( price_account.try_borrow_mut_data()?.as_mut_ptr(), - clock.slot, - clock.unix_timestamp, + slots_since_last_update as i64, // Ensure slots_since_last_update is cast to i64, as expected by the function signature ); } } diff --git a/program/rust/src/tests/test_upd_aggregate.rs b/program/rust/src/tests/test_upd_aggregate.rs index 12808955..7841cde0 100644 --- a/program/rust/src/tests/test_upd_aggregate.rs +++ b/program/rust/src/tests/test_upd_aggregate.rs @@ -67,7 +67,6 @@ fn test_upd_aggregate() { corp_act_status_: 0, }; - let mut instruction_data = [0u8; size_of::()]; populate_instruction(&mut instruction_data, 42, 2, 1); @@ -109,17 +108,57 @@ fn test_upd_aggregate() { assert_eq!(price_data.prev_timestamp_, 0); } + // test that same slot aggregate works but for twap and twac it should be based on the previous slot data and not the current slot + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.num_ = 2; + price_data.comp_[1].latest_ = p2; + } + + unsafe { + assert!(c_upd_aggregate( + price_account.try_borrow_mut_data().unwrap().as_mut_ptr(), + 1001, + 2, + )); + } + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + + assert_eq!(price_data.agg_.price_, 145); + assert_eq!(price_data.agg_.conf_, 55); + assert_eq!(price_data.twap_.val_, 100); + assert_eq!(price_data.twac_.val_, 10); + assert_eq!(price_data.num_qt_, 2); + assert_eq!(price_data.timestamp_, 2); + assert_eq!(price_data.prev_slot_, 1000); + assert_eq!(price_data.prev_price_, 100); + assert_eq!(price_data.prev_conf_, 10); + assert_eq!(price_data.prev_timestamp_, 1); + } + // single publisher { let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); price_data.num_ = 1; + price_data.num_qt_ = 0; price_data.last_slot_ = 1000; + price_data.twap_.val_ = 0; + price_data.twap_.numer_ = 0; + price_data.twap_.denom_ = 0; + price_data.twac_.val_ = 0; + price_data.twac_.numer_ = 0; + price_data.twac_.denom_ = 0; + price_data.timestamp_ = 0; price_data.agg_.pub_slot_ = 1000; price_data.comp_[0].latest_ = p1; price_data.prev_slot_ = 0; price_data.prev_price_ = 0; price_data.prev_conf_ = 0; price_data.prev_timestamp_ = 0; + price_data.agg_.price_ = 0; + price_data.agg_.conf_ = 0; price_data.agg_.status_ = PC_STATUS_UNKNOWN; } @@ -136,8 +175,8 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.price_, 100); assert_eq!(price_data.agg_.conf_, 10); - assert_eq!(price_data.twap_.val_, 100); - assert_eq!(price_data.twac_.val_, 10); + assert_eq!(price_data.twap_.val_, 0); + assert_eq!(price_data.twac_.val_, 0); assert_eq!(price_data.num_qt_, 1); assert_eq!(price_data.timestamp_, 1); assert_eq!(price_data.prev_slot_, 0); From e88c18715b97fd3adecd3e244bc6cf39b9d253a8 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 14 Mar 2024 13:04:31 +0900 Subject: [PATCH 05/60] remove twap and twac from test_up_aggregate --- program/rust/src/tests/test_upd_aggregate.rs | 61 +------------------- 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/program/rust/src/tests/test_upd_aggregate.rs b/program/rust/src/tests/test_upd_aggregate.rs index 7841cde0..3e24e3b8 100644 --- a/program/rust/src/tests/test_upd_aggregate.rs +++ b/program/rust/src/tests/test_upd_aggregate.rs @@ -77,7 +77,7 @@ fn test_upd_aggregate() { price_account.is_signer = false; PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); - // test same slot aggregation, aggregate price and conf should be updated but not twap and twac + // test same slot aggregation, aggregate price and conf should be updated { let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); price_data.num_ = 1; @@ -93,63 +93,12 @@ fn test_upd_aggregate() { )); } - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - - assert_eq!(price_data.agg_.price_, 100); - assert_eq!(price_data.agg_.conf_, 10); - assert_eq!(price_data.twap_.val_, 0); - assert_eq!(price_data.twac_.val_, 0); - assert_eq!(price_data.num_qt_, 1); - assert_eq!(price_data.timestamp_, 1); - assert_eq!(price_data.prev_slot_, 0); - assert_eq!(price_data.prev_price_, 0); - assert_eq!(price_data.prev_conf_, 0); - assert_eq!(price_data.prev_timestamp_, 0); - } - - // test that same slot aggregate works but for twap and twac it should be based on the previous slot data and not the current slot - { - let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - price_data.num_ = 2; - price_data.comp_[1].latest_ = p2; - } - - unsafe { - assert!(c_upd_aggregate( - price_account.try_borrow_mut_data().unwrap().as_mut_ptr(), - 1001, - 2, - )); - } - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - - assert_eq!(price_data.agg_.price_, 145); - assert_eq!(price_data.agg_.conf_, 55); - assert_eq!(price_data.twap_.val_, 100); - assert_eq!(price_data.twac_.val_, 10); - assert_eq!(price_data.num_qt_, 2); - assert_eq!(price_data.timestamp_, 2); - assert_eq!(price_data.prev_slot_, 1000); - assert_eq!(price_data.prev_price_, 100); - assert_eq!(price_data.prev_conf_, 10); - assert_eq!(price_data.prev_timestamp_, 1); - } - // single publisher { let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); price_data.num_ = 1; price_data.num_qt_ = 0; price_data.last_slot_ = 1000; - price_data.twap_.val_ = 0; - price_data.twap_.numer_ = 0; - price_data.twap_.denom_ = 0; - price_data.twac_.val_ = 0; - price_data.twac_.numer_ = 0; - price_data.twac_.denom_ = 0; price_data.timestamp_ = 0; price_data.agg_.pub_slot_ = 1000; price_data.comp_[0].latest_ = p1; @@ -175,8 +124,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.price_, 100); assert_eq!(price_data.agg_.conf_, 10); - assert_eq!(price_data.twap_.val_, 0); - assert_eq!(price_data.twac_.val_, 0); assert_eq!(price_data.num_qt_, 1); assert_eq!(price_data.timestamp_, 1); assert_eq!(price_data.prev_slot_, 0); @@ -209,8 +156,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.price_, 145); assert_eq!(price_data.agg_.conf_, 55); - assert_eq!(price_data.twap_.val_, 106); - assert_eq!(price_data.twac_.val_, 16); assert_eq!(price_data.num_qt_, 2); assert_eq!(price_data.timestamp_, 2); assert_eq!(price_data.prev_slot_, 1000); @@ -244,8 +189,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.price_, 200); assert_eq!(price_data.agg_.conf_, 90); - assert_eq!(price_data.twap_.val_, 114); - assert_eq!(price_data.twac_.val_, 23); assert_eq!(price_data.num_qt_, 3); assert_eq!(price_data.timestamp_, 3); assert_eq!(price_data.prev_slot_, 1000); @@ -280,8 +223,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.price_, 245); assert_eq!(price_data.agg_.conf_, 85); - assert_eq!(price_data.twap_.val_, 125); - assert_eq!(price_data.twac_.val_, 28); assert_eq!(price_data.num_qt_, 4); assert_eq!(price_data.timestamp_, 4); assert_eq!(price_data.last_slot_, 1001); From 2216b043bf82e64a9d60ed4e61593facd75f4c8d Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 14 Mar 2024 13:08:59 +0900 Subject: [PATCH 06/60] fix test_sizes --- program/rust/src/tests/test_sizes.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/program/rust/src/tests/test_sizes.rs b/program/rust/src/tests/test_sizes.rs index 153e8235..b3ec7507 100644 --- a/program/rust/src/tests/test_sizes.rs +++ b/program/rust/src/tests/test_sizes.rs @@ -69,6 +69,9 @@ fn test_sizes() { + 3 * size_of::() + size_of::() + (PC_NUM_COMP_PYTHNET as usize) * size_of::() + + size_of::() + + size_of::() + + size_of::() + size_of::() ); assert_eq!(size_of::(), 12576); From 29403a92ea05d4931a82e219f12f7dec182e1197 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 14 Mar 2024 15:21:27 +0900 Subject: [PATCH 07/60] fix borrow reference bug --- program/rust/src/processor/upd_price.rs | 45 ++++++++++++++++--------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index c1879388..754f29cd 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -180,24 +180,39 @@ pub fn upd_price( // Try to update the aggregate #[allow(unused_variables)] - let aggregate_updated = unsafe { - c_upd_aggregate( - price_account.try_borrow_mut_data()?.as_mut_ptr(), - clock.slot, - clock.unix_timestamp, - ) - }; + let mut aggregate_updated = false; + + // NOTE: c_upd_aggregate must use a raw pointer to price + // data. Solana's `.borrow_*` methods require exclusive + // access, i.e. no other borrow can exist for the account. + #[allow(unused_assignments)] + if slots_since_last_update > 0 { + unsafe { + aggregate_updated = c_upd_aggregate( + price_account.try_borrow_mut_data()?.as_mut_ptr(), + clock.slot, + clock.unix_timestamp, + ); + } + } if aggregate_updated { - // get price data here - let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - // check if its the first price update in the slot - if slots_since_last_update > 0 { - price_data.prev_twap_ = price_data.twap_; - price_data.prev_twac_ = price_data.twac_; + // The price_data borrow happens in a scope because it must be + // dropped before we borrow again as raw data pointer for the C + // aggregation logic. + #[cfg(feature = "pythnet")] + { + // get price data here + let mut price_data = + load_checked::(price_account, cmd_args.header.version)?; + // check if its the first price update in the slot + if slots_since_last_update > 0 { + price_data.prev_twap_ = price_data.twap_; + price_data.prev_twac_ = price_data.twac_; + } + price_data.twap_ = price_data.prev_twap_; + price_data.twac_ = price_data.prev_twac_; } - price_data.twap_ = price_data.prev_twap_; - price_data.twac_ = price_data.prev_twac_; unsafe { c_upd_twap( price_account.try_borrow_mut_data()?.as_mut_ptr(), From 3dfbc75fd2a3b4eb1866788368b922aabd82bf40 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 15 Mar 2024 13:19:51 +0900 Subject: [PATCH 08/60] fix build --- program/c/src/oracle/upd_aggregate.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index e2ff29ea..5caba458 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -139,9 +139,6 @@ static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest return false; } - // get number of slots from last published valid price - int64_t agg_diff = ( int64_t )slot - ( int64_t )( ptr->last_slot_ ); - // Update the value of the previous price, if it had TRADING status. if ( ptr->agg_.status_ == PC_STATUS_TRADING ) { ptr->prev_slot_ = ptr->agg_.pub_slot_; From a756c22bd6c12e916cb0aafbaa3bec594b4b4391 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 18 Mar 2024 17:16:21 +0900 Subject: [PATCH 09/60] fix logic --- program/c/src/oracle/upd_aggregate.h | 5 -- program/rust/src/processor/upd_price.rs | 62 ++++++++++++++++----- program/rust/src/tests/test_ema.rs | 3 - program/rust/src/tests/test_upd_price.rs | 70 ++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 22 deletions(-) diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index 5caba458..eaf54a33 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -134,11 +134,6 @@ static inline void upd_twap( // update aggregate price static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timestamp ) { - // only re-compute aggregate in current or future slots - if ( slot < ptr->agg_.pub_slot_ ) { - return false; - } - // Update the value of the previous price, if it had TRADING status. if ( ptr->agg_.status_ == PC_STATUS_TRADING ) { ptr->prev_slot_ = ptr->agg_.pub_slot_; diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 754f29cd..604a6ad2 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -144,6 +144,7 @@ pub fn upd_price( let mut publisher_index: usize = 0; let latest_aggregate_price: PriceInfo; + #[cfg(feature = "pythnet")] // The price_data borrow happens in a scope because it must be // dropped before we borrow again as raw data pointer for the C // aggregation logic. @@ -175,26 +176,51 @@ pub fn upd_price( )?; } - // get number of slots from last update - let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; + { + // Reload price data as a struct after c_upd_aggregate() borrow is dropped + let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - // Try to update the aggregate - #[allow(unused_variables)] - let mut aggregate_updated = false; + // Try to update the publisher's price + if is_component_update(cmd_args)? { + // IMPORTANT: If the publisher does not meet the price/conf + // ratio condition, its price will not count for the next + // aggregate. + let status: u32 = get_status_for_conf_price_ratio( + cmd_args.price, + cmd_args.confidence, + cmd_args.status, + )?; + + { + let publisher_price = &mut price_data.comp_[publisher_index].latest_; + publisher_price.price_ = cmd_args.price; + publisher_price.conf_ = cmd_args.confidence; + publisher_price.status_ = status; + publisher_price.pub_slot_ = cmd_args.publishing_slot; + } + } + } - // NOTE: c_upd_aggregate must use a raw pointer to price - // data. Solana's `.borrow_*` methods require exclusive - // access, i.e. no other borrow can exist for the account. - #[allow(unused_assignments)] - if slots_since_last_update > 0 { + let should_call_c_upd_aggregate: bool = if cfg!(feature = "pythnet") { + clock.slot >= latest_aggregate_price.pub_slot_ + } else { + clock.slot > latest_aggregate_price.pub_slot_ + }; + + let aggregate_updated = if should_call_c_upd_aggregate { unsafe { - aggregate_updated = c_upd_aggregate( + c_upd_aggregate( price_account.try_borrow_mut_data()?.as_mut_ptr(), clock.slot, clock.unix_timestamp, - ); + ) } - } + } else { + false // Do not call c_upd_aggregate and set aggregate_updated to false + }; + + // get number of slots from last update + let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; if aggregate_updated { // The price_data borrow happens in a scope because it must be @@ -209,15 +235,23 @@ pub fn upd_price( if slots_since_last_update > 0 { price_data.prev_twap_ = price_data.twap_; price_data.prev_twac_ = price_data.twac_; + price_data.prev_price_cumulative = price_data.price_cumulative; } price_data.twap_ = price_data.prev_twap_; price_data.twac_ = price_data.prev_twac_; + price_data.price_cumulative = price_data.prev_price_cumulative; } unsafe { c_upd_twap( price_account.try_borrow_mut_data()?.as_mut_ptr(), slots_since_last_update as i64, // Ensure slots_since_last_update is cast to i64, as expected by the function signature ); + #[cfg(feature = "pythnet")] + { + let mut price_data = + load_checked::(price_account, cmd_args.header.version)?; + price_data.update_price_cumulative()?; + } } } @@ -233,7 +267,6 @@ pub fn upd_price( // will send the message. if aggregate_updated { price_data.message_sent_ = 0; - price_data.update_price_cumulative()?; } if let Some(accumulator_accounts) = maybe_accumulator_accounts { @@ -297,6 +330,7 @@ pub fn upd_price( } } + #[cfg(not(feature = "pythnet"))] // Try to update the publisher's price if is_component_update(cmd_args)? { // IMPORTANT: If the publisher does not meet the price/conf diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index 1a1cae10..73b15f2c 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -18,7 +18,6 @@ use { test_generator::test_resources, }; - #[test_resources("program/rust/test_data/ema/*.csv")] fn test_ema(input_path_raw: &str) { let (inputs, expected_outputs) = read_test_data(input_path_raw); @@ -110,7 +109,6 @@ fn run_ema_test(inputs: &[InputRecord], expected_outputs: &[OutputRecord]) { } } - // TODO: put these functions somewhere more accessible pub fn upd_aggregate( price_account: &mut PriceAccount, @@ -130,7 +128,6 @@ pub fn upd_twap(price_account: &mut PriceAccount, nslots: i64) { unsafe { c_upd_twap((price_account as *mut PriceAccount) as *mut u8, nslots) } } - #[derive(Serialize, Deserialize, Debug)] struct InputRecord { price: i64, diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 9726cda7..a92986b5 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -70,6 +70,7 @@ fn test_upd_price() { ) .is_ok()); + #[cfg(not(feature = "pythnet"))] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 42); @@ -82,6 +83,19 @@ fn test_upd_price() { assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); } + #[cfg(feature = "pythnet")] + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 42); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 0); + assert_eq!(price_data.agg_.pub_slot_, 1); + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + // add some prices for current slot - get rejected populate_instruction(&mut instruction_data, 43, 2, 1); @@ -98,6 +112,7 @@ fn test_upd_price() { Err(ProgramError::InvalidArgument) ); + #[cfg(not(feature = "pythnet"))] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 42); @@ -110,6 +125,19 @@ fn test_upd_price() { assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); } + #[cfg(feature = "pythnet")] + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 42); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 0); + assert_eq!(price_data.agg_.pub_slot_, 1); + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + // add next price in new slot triggering snapshot and aggregate calc populate_instruction(&mut instruction_data, 81, 2, 2); update_clock_slot(&mut clock_account, 3); @@ -125,6 +153,7 @@ fn test_upd_price() { ) .is_ok()); + #[cfg(not(feature = "pythnet"))] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 81); @@ -137,6 +166,19 @@ fn test_upd_price() { assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); } + #[cfg(feature = "pythnet")] + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 2); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 1); + assert_eq!(price_data.agg_.pub_slot_, 3); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + // next price doesn't change but slot does populate_instruction(&mut instruction_data, 81, 2, 3); update_clock_slot(&mut clock_account, 4); @@ -239,6 +281,7 @@ fn test_upd_price() { ) .is_ok()); + #[cfg(not(feature = "pythnet"))] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 50); @@ -251,6 +294,19 @@ fn test_upd_price() { assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); } + #[cfg(feature = "pythnet")] + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 50); + assert_eq!(price_data.comp_[0].latest_.conf_, 20); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 5); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); + assert_eq!(price_data.valid_slot_, 5); + assert_eq!(price_data.agg_.pub_slot_, 6); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); // in pythnet aggregation is on the same slot so status turns to unknown + } + // Crank one more time and aggregate should be unknown populate_instruction(&mut instruction_data, 50, 20, 6); update_clock_slot(&mut clock_account, 7); @@ -293,6 +349,7 @@ fn test_upd_price() { ) .is_ok()); + #[cfg(not(feature = "pythnet"))] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, -100); @@ -305,6 +362,19 @@ fn test_upd_price() { assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); } + #[cfg(feature = "pythnet")] + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, -100); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 7); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 7); + assert_eq!(price_data.agg_.pub_slot_, 8); + assert_eq!(price_data.agg_.price_, -100); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + // Crank again for aggregate populate_instruction(&mut instruction_data, -100, 1, 8); update_clock_slot(&mut clock_account, 9); From 8922a9c5bf0fb4b502658bef9dda6ddfa9a15488 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 19 Mar 2024 11:10:41 +0900 Subject: [PATCH 10/60] fix test_publish --- program/rust/src/tests/test_publish.rs | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/program/rust/src/tests/test_publish.rs b/program/rust/src/tests/test_publish.rs index 208642a4..2487fdb6 100644 --- a/program/rust/src/tests/test_publish.rs +++ b/program/rust/src/tests/test_publish.rs @@ -64,6 +64,7 @@ async fn test_publish() { .await .unwrap(); + #[cfg(not(feature = "pythnet"))] { let price_data = sim .get_account_data_as::(price) @@ -79,6 +80,22 @@ async fn test_publish() { assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); } + #[cfg(feature = "pythnet")] + { + let price_data = sim + .get_account_data_as::(price) + .await + .unwrap(); + + assert_eq!(price_data.comp_[0].latest_.price_, 150); + assert_eq!(price_data.comp_[0].latest_.conf_, 7); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + + assert_eq!(price_data.comp_[0].agg_.price_, 150); + assert_eq!(price_data.comp_[0].agg_.conf_, 7); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); + } + sim.warp_to_slot(2).await.unwrap(); sim.upd_price( &publisher, @@ -92,6 +109,7 @@ async fn test_publish() { .await .unwrap(); + #[cfg(not(feature = "pythnet"))] { let price_data = sim .get_account_data_as::(price) @@ -106,4 +124,20 @@ async fn test_publish() { assert_eq!(price_data.comp_[0].agg_.conf_, 7); assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); } + + #[cfg(feature = "pythnet")] + { + let price_data = sim + .get_account_data_as::(price) + .await + .unwrap(); + + assert_eq!(price_data.comp_[0].latest_.price_, 0); + assert_eq!(price_data.comp_[0].latest_.conf_, 0); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_UNKNOWN); + + assert_eq!(price_data.comp_[0].agg_.price_, 0); + assert_eq!(price_data.comp_[0].agg_.conf_, 0); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); + } } From 760f5ebaf705069e75889d3d0fd1e91d3820d3f9 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 19 Mar 2024 11:27:03 +0900 Subject: [PATCH 11/60] format --- program/rust/src/tests/test_publish.rs | 65 +++++++++++--------------- 1 file changed, 26 insertions(+), 39 deletions(-) diff --git a/program/rust/src/tests/test_publish.rs b/program/rust/src/tests/test_publish.rs index 2487fdb6..3ffc52ad 100644 --- a/program/rust/src/tests/test_publish.rs +++ b/program/rust/src/tests/test_publish.rs @@ -75,25 +75,19 @@ async fn test_publish() { assert_eq!(price_data.comp_[0].latest_.conf_, 7); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.comp_[0].agg_.price_, 0); - assert_eq!(price_data.comp_[0].agg_.conf_, 0); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); - } - - #[cfg(feature = "pythnet")] - { - let price_data = sim - .get_account_data_as::(price) - .await - .unwrap(); - - assert_eq!(price_data.comp_[0].latest_.price_, 150); - assert_eq!(price_data.comp_[0].latest_.conf_, 7); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.comp_[0].agg_.price_, 150); - assert_eq!(price_data.comp_[0].agg_.conf_, 7); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); + #[cfg(not(feature = "pythnet"))] + { + assert_eq!(price_data.comp_[0].agg_.price_, 0); + assert_eq!(price_data.comp_[0].agg_.conf_, 0); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); + } + + #[cfg(feature = "pythnet")] + { + assert_eq!(price_data.comp_[0].agg_.price_, 150); + assert_eq!(price_data.comp_[0].agg_.conf_, 7); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); + } } sim.warp_to_slot(2).await.unwrap(); @@ -109,23 +103,6 @@ async fn test_publish() { .await .unwrap(); - #[cfg(not(feature = "pythnet"))] - { - let price_data = sim - .get_account_data_as::(price) - .await - .unwrap(); - - assert_eq!(price_data.comp_[0].latest_.price_, 0); - assert_eq!(price_data.comp_[0].latest_.conf_, 0); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_UNKNOWN); - - assert_eq!(price_data.comp_[0].agg_.price_, 150); - assert_eq!(price_data.comp_[0].agg_.conf_, 7); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); - } - - #[cfg(feature = "pythnet")] { let price_data = sim .get_account_data_as::(price) @@ -136,8 +113,18 @@ async fn test_publish() { assert_eq!(price_data.comp_[0].latest_.conf_, 0); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_UNKNOWN); - assert_eq!(price_data.comp_[0].agg_.price_, 0); - assert_eq!(price_data.comp_[0].agg_.conf_, 0); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); + #[cfg(not(feature = "pythnet"))] + { + assert_eq!(price_data.comp_[0].agg_.price_, 150); + assert_eq!(price_data.comp_[0].agg_.conf_, 7); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); + } + + #[cfg(feature = "pythnet")] + { + assert_eq!(price_data.comp_[0].agg_.price_, 0); + assert_eq!(price_data.comp_[0].agg_.conf_, 0); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); + } } } From dc2f0e709d5e1a0368d04100e389ab8c8be98017 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 19 Mar 2024 11:44:33 +0900 Subject: [PATCH 12/60] fix test_publish_batch --- program/rust/src/tests/test_publish_batch.rs | 51 ++++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/program/rust/src/tests/test_publish_batch.rs b/program/rust/src/tests/test_publish_batch.rs index f1c5424b..5f6b8f05 100644 --- a/program/rust/src/tests/test_publish_batch.rs +++ b/program/rust/src/tests/test_publish_batch.rs @@ -84,9 +84,22 @@ async fn test_publish_batch() { price_data.comp_[0].latest_.status_, get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status).unwrap() ); - assert_eq!(price_data.comp_[0].agg_.price_, 0); - assert_eq!(price_data.comp_[0].agg_.conf_, 0); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); + #[cfg(not(feature = "pythnet"))] + { + assert_eq!(price_data.comp_[0].agg_.price_, 0); + assert_eq!(price_data.comp_[0].agg_.conf_, 0); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); + } + #[cfg(feature = "pythnet")] + { + assert_eq!(price_data.comp_[0].agg_.price_, quote.price); + assert_eq!(price_data.comp_[0].agg_.conf_, quote.confidence); + assert_eq!( + price_data.comp_[0].agg_.status_, + get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status) + .unwrap() + ); + } } sim.warp_to_slot(2).await.unwrap(); @@ -111,7 +124,6 @@ async fn test_publish_batch() { .get_account_data_as::(*price) .await .unwrap(); - let quote = quotes.get(key).unwrap(); let new_quote = new_quotes.get(key).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, new_quote.price); @@ -125,11 +137,30 @@ async fn test_publish_batch() { ) .unwrap() ); - assert_eq!(price_data.comp_[0].agg_.price_, quote.price); - assert_eq!(price_data.comp_[0].agg_.conf_, quote.confidence); - assert_eq!( - price_data.comp_[0].agg_.status_, - get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status).unwrap() - ); + #[cfg(not(feature = "pythnet"))] + { + let quote = quotes.get(key).unwrap(); + assert_eq!(price_data.comp_[0].agg_.price_, quote.price); + assert_eq!(price_data.comp_[0].agg_.conf_, quote.confidence); + assert_eq!( + price_data.comp_[0].agg_.status_, + get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status) + .unwrap() + ); + } + #[cfg(feature = "pythnet")] + { + assert_eq!(price_data.comp_[0].agg_.price_, new_quote.price); + assert_eq!(price_data.comp_[0].agg_.conf_, new_quote.confidence); + assert_eq!( + price_data.comp_[0].agg_.status_, + get_status_for_conf_price_ratio( + new_quote.price, + new_quote.confidence, + new_quote.status + ) + .unwrap() + ); + } } } From 89a8e7a9cd8463cae8e2056794f098898e7c953f Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 19 Mar 2024 11:49:31 +0900 Subject: [PATCH 13/60] fix test_upd_price_no_fail_on_error --- .../tests/test_upd_price_no_fail_on_error.rs | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/program/rust/src/tests/test_upd_price_no_fail_on_error.rs b/program/rust/src/tests/test_upd_price_no_fail_on_error.rs index 3483ad1d..a3ebed02 100644 --- a/program/rust/src/tests/test_upd_price_no_fail_on_error.rs +++ b/program/rust/src/tests/test_upd_price_no_fail_on_error.rs @@ -52,7 +52,6 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { update_clock_slot(&mut clock_account, 1); - // Check that the normal upd_price fails populate_instruction(&mut instruction_data, 42, 9, 1, true); @@ -69,7 +68,6 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { Err(OracleError::PermissionViolation.into()) ); - populate_instruction(&mut instruction_data, 42, 9, 1, false); // We haven't permissioned the publish account for the price account // yet, so any update should fail silently and have no effect. The @@ -85,7 +83,6 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { ) .is_ok()); - { let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 0); @@ -122,8 +119,16 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + #[cfg(not(feature = "pythnet"))] + { + assert_eq!(price_data.agg_.price_, 0); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + } + #[cfg(feature = "pythnet")] + { + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } } // Invalid updates, such as publishing an update for the current slot, @@ -164,12 +169,19 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + #[cfg(not(feature = "pythnet"))] + { + assert_eq!(price_data.agg_.price_, 0); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + } + #[cfg(feature = "pythnet")] + { + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } } } - // Create an upd_price_no_fail_on_error or upd_price instruction with the provided parameters fn populate_instruction( instruction_data: &mut [u8], From 59359f274bf8a3133b64bc0470fe3fdc2837c754 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 19 Mar 2024 17:16:55 +0900 Subject: [PATCH 14/60] fix test_upd_price_v2 --- program/rust/src/tests/test_upd_price_v2.rs | 97 ++++++++++----------- 1 file changed, 44 insertions(+), 53 deletions(-) diff --git a/program/rust/src/tests/test_upd_price_v2.rs b/program/rust/src/tests/test_upd_price_v2.rs index 0e0c1f4b..22951f8d 100644 --- a/program/rust/src/tests/test_upd_price_v2.rs +++ b/program/rust/src/tests/test_upd_price_v2.rs @@ -77,12 +77,11 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.conf_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - - assert_eq!(price_data.price_cumulative.price, 0); - assert_eq!(price_data.price_cumulative.conf, 0); + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.conf_, 2); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.price_cumulative.price, 42); + assert_eq!(price_data.price_cumulative.conf, 2); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -110,12 +109,11 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.conf_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - - assert_eq!(price_data.price_cumulative.price, 0); - assert_eq!(price_data.price_cumulative.conf, 0); + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.conf_, 2); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.price_cumulative.price, 42); + assert_eq!(price_data.price_cumulative.conf, 2); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -141,11 +139,10 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 1); assert_eq!(price_data.agg_.pub_slot_, 3); - assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.conf_, 2); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.price_cumulative.price, 3 * 42); + assert_eq!(price_data.price_cumulative.price, 42 + 2 * 81); assert_eq!(price_data.price_cumulative.conf, 3 * 2); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -174,9 +171,8 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.conf_, 2); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2); + assert_eq!(price_data.price_cumulative.price, 42 + 3 * 81); + assert_eq!(price_data.price_cumulative.conf, 4 * 2); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -204,9 +200,8 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.conf_, 2); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 2); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 2); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 5 * 2); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -237,9 +232,8 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.conf_, 2); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 2); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 2); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 5 * 2); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -274,10 +268,9 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.pub_slot_, 6); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 3); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 3); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 2 * 5); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -306,9 +299,8 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.conf_, 2); assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 3); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 3); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 2 * 5); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -334,12 +326,11 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 7); assert_eq!(price_data.agg_.pub_slot_, 8); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 3); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 3); + assert_eq!(price_data.agg_.price_, -100); + assert_eq!(price_data.agg_.conf_, 1); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 3); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 3); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -368,9 +359,8 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.price_, -100); assert_eq!(price_data.agg_.conf_, 1); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 3 - 100 * 3); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 3 + 3); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 4); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 4); assert_eq!(price_data.price_cumulative.num_down_slots, 0); } @@ -397,13 +387,15 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 9); assert_eq!(price_data.agg_.pub_slot_, 50); - assert_eq!(price_data.agg_.price_, -100); - assert_eq!(price_data.agg_.conf_, 1); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - - assert_eq!(price_data.price_cumulative.price, 3 * 42 + 81 * 3 - 100 * 3); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 3 + 3); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); + assert_eq!(price_data.agg_.price_, 60); + assert_eq!(price_data.agg_.conf_, 4); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 4 + 60 * 41 + ); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 4 + 41 * 4); + assert_eq!(price_data.price_cumulative.num_down_slots, 16); } // Crank again for aggregate @@ -429,16 +421,15 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 50); assert_eq!(price_data.agg_.pub_slot_, 51); - assert_eq!(price_data.agg_.price_, 60); - assert_eq!(price_data.agg_.conf_, 4); + assert_eq!(price_data.agg_.price_, 55); + assert_eq!(price_data.agg_.conf_, 5); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!( price_data.price_cumulative.price, - 3 * 42 + 81 * 3 - 100 * 3 + 42 * 60 + 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 ); - assert_eq!(price_data.price_cumulative.conf, 3 * 2 + 2 * 3 + 3 + 42 * 4); - assert_eq!(price_data.price_cumulative.num_down_slots, 17); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 4 + 41 * 4 + 5); + assert_eq!(price_data.price_cumulative.num_down_slots, 16); } Ok(()) From b24cb442b403ff4d5fbdadd643353dabb98f8d03 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 13:06:33 +0900 Subject: [PATCH 15/60] fix logic --- program/rust/src/processor/upd_price.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 604a6ad2..fa56be25 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -144,7 +144,6 @@ pub fn upd_price( let mut publisher_index: usize = 0; let latest_aggregate_price: PriceInfo; - #[cfg(feature = "pythnet")] // The price_data borrow happens in a scope because it must be // dropped before we borrow again as raw data pointer for the C // aggregation logic. @@ -176,6 +175,7 @@ pub fn upd_price( )?; } + #[cfg(feature = "pythnet")] { // Reload price data as a struct after c_upd_aggregate() borrow is dropped let mut price_data = load_checked::(price_account, cmd_args.header.version)?; From cb96ccad2dcbc362d669416f3bdde834e526b9aa Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 13:08:22 +0900 Subject: [PATCH 16/60] refactor --- program/rust/src/processor/upd_price.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index fa56be25..2572c9c3 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -240,18 +240,13 @@ pub fn upd_price( price_data.twap_ = price_data.prev_twap_; price_data.twac_ = price_data.prev_twac_; price_data.price_cumulative = price_data.prev_price_cumulative; + price_data.update_price_cumulative()?; } unsafe { c_upd_twap( price_account.try_borrow_mut_data()?.as_mut_ptr(), slots_since_last_update as i64, // Ensure slots_since_last_update is cast to i64, as expected by the function signature ); - #[cfg(feature = "pythnet")] - { - let mut price_data = - load_checked::(price_account, cmd_args.header.version)?; - price_data.update_price_cumulative()?; - } } } From c0ddc2856e2b98034ea021557494e8f600c7ab8b Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 13:22:31 +0900 Subject: [PATCH 17/60] update function desc --- program/rust/src/processor/upd_price.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 2572c9c3..6279e39d 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -88,9 +88,9 @@ pub fn upd_price_no_fail_on_error( } } -/// Update a publisher's price for the provided product. If this update is -/// the first update in a slot, this operation will also trigger price aggregation -/// and result in a new aggregate price in the account. +/// Update a publisher's price for the provided product. This operation's behavior varies based on the feature flag: +/// - For `feature = "pythnet"`, price aggregation will trigger on the same slot for every price update, resulting in a new aggregate price in the account. +/// - For non-`pythnet` configurations, the first price update in a slot triggers price aggregation for the prices of the previous slot, resulting in a new aggregate price in the account. /// /// account[0] the publisher's account (funds the tx) [signer writable] /// fails if the publisher's public key is not permissioned for the price account. From 1605d947dcbda1336b1b914956b508df95275200 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 15:06:46 +0900 Subject: [PATCH 18/60] fix logic --- program/c/src/oracle/upd_aggregate.h | 8 --- program/rust/src/processor/upd_price.rs | 15 ++++-- program/rust/src/tests/test_upd_price_v2.rs | 58 +++++++++++++++++++++ 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index eaf54a33..ab77a070 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -134,14 +134,6 @@ static inline void upd_twap( // update aggregate price static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timestamp ) { - // Update the value of the previous price, if it had TRADING status. - if ( ptr->agg_.status_ == PC_STATUS_TRADING ) { - ptr->prev_slot_ = ptr->agg_.pub_slot_; - ptr->prev_price_ = ptr->agg_.price_; - ptr->prev_conf_ = ptr->agg_.conf_; - ptr->prev_timestamp_ = ptr->timestamp_; - } - // update aggregate details ready for next slot ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 6279e39d..180d63ec 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -16,6 +16,7 @@ use { PriceAccount, PriceInfo, }, + c_oracle_header::PC_STATUS_TRADING, deserialize::{ load, load_checked, @@ -207,7 +208,18 @@ pub fn upd_price( clock.slot > latest_aggregate_price.pub_slot_ }; + // get number of slots from last update + let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; + let aggregate_updated = if should_call_c_upd_aggregate { + if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { + let mut price_data = + load_checked::(price_account, cmd_args.header.version)?; + price_data.prev_slot_ = price_data.agg_.pub_slot_; + price_data.prev_price_ = price_data.agg_.price_; + price_data.prev_conf_ = price_data.agg_.conf_; + price_data.prev_timestamp_ = clock.unix_timestamp; + } unsafe { c_upd_aggregate( price_account.try_borrow_mut_data()?.as_mut_ptr(), @@ -219,9 +231,6 @@ pub fn upd_price( false // Do not call c_upd_aggregate and set aggregate_updated to false }; - // get number of slots from last update - let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; - if aggregate_updated { // The price_data borrow happens in a scope because it must be // dropped before we borrow again as raw data pointer for the C diff --git a/program/rust/src/tests/test_upd_price_v2.rs b/program/rust/src/tests/test_upd_price_v2.rs index 22951f8d..3b6b84d7 100644 --- a/program/rust/src/tests/test_upd_price_v2.rs +++ b/program/rust/src/tests/test_upd_price_v2.rs @@ -432,6 +432,64 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.price_cumulative.num_down_slots, 16); } + let mut funding_setup_two = AccountSetup::new_funding(); + let funding_account_two = funding_setup_two.as_account_info(); + + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.num_ = 2; + price_data.comp_[1].pub_ = *funding_account_two.key; + } + + populate_instruction(&mut instruction_data, 10, 1, 100); + update_clock_slot(&mut clock_account, 100); + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + populate_instruction(&mut instruction_data, 20, 2, 100); + process_instruction( + &program_id, + &[ + funding_account_two.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 10); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 100); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.comp_[1].latest_.price_, 20); + assert_eq!(price_data.comp_[1].latest_.conf_, 2); + assert_eq!(price_data.comp_[1].latest_.pub_slot_, 100); + assert_eq!(price_data.comp_[1].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 100); + assert_eq!(price_data.agg_.pub_slot_, 100); + assert_eq!(price_data.agg_.price_, 14); + assert_eq!(price_data.agg_.conf_, 6); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 + 14 * 49 + ); + assert_eq!( + price_data.price_cumulative.conf, + 2 * 5 + 4 + 41 * 4 + 5 + 6 * 49 + ); + assert_eq!(price_data.price_cumulative.num_down_slots, 40); // prev num_down_slots was 16 and since pub slot is 100 and last pub slot was 51, slot_gap is 49 and default latency is 25, so num_down_slots = 49 - 25 = 24, so total num_down_slots = 16 + 24 = 40 + } + Ok(()) } From 9b1b7e0cf87e2dae0769e158fb8fd2f3f56600ba Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 15:08:42 +0900 Subject: [PATCH 19/60] add comments --- program/rust/src/tests/test_upd_price_v2.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/program/rust/src/tests/test_upd_price_v2.rs b/program/rust/src/tests/test_upd_price_v2.rs index 3b6b84d7..daaa0f88 100644 --- a/program/rust/src/tests/test_upd_price_v2.rs +++ b/program/rust/src/tests/test_upd_price_v2.rs @@ -432,6 +432,7 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.price_cumulative.num_down_slots, 16); } + // add new test for multiple publishers and ensure that price_cumulative is updated correctly let mut funding_setup_two = AccountSetup::new_funding(); let funding_account_two = funding_setup_two.as_account_info(); From ddfc4ab07ca935afad4ab0a40d30a08727939cf3 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 18:52:45 +0900 Subject: [PATCH 20/60] fix tests --- program/rust/src/tests/test_upd_aggregate.rs | 36 -------------------- program/rust/src/tests/test_upd_price_v2.rs | 5 ++- 2 files changed, 4 insertions(+), 37 deletions(-) diff --git a/program/rust/src/tests/test_upd_aggregate.rs b/program/rust/src/tests/test_upd_aggregate.rs index 3e24e3b8..2747e1c7 100644 --- a/program/rust/src/tests/test_upd_aggregate.rs +++ b/program/rust/src/tests/test_upd_aggregate.rs @@ -158,10 +158,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.conf_, 55); assert_eq!(price_data.num_qt_, 2); assert_eq!(price_data.timestamp_, 2); - assert_eq!(price_data.prev_slot_, 1000); - assert_eq!(price_data.prev_price_, 100); - assert_eq!(price_data.prev_conf_, 10); - assert_eq!(price_data.prev_timestamp_, 1); } // three publishers @@ -191,10 +187,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.conf_, 90); assert_eq!(price_data.num_qt_, 3); assert_eq!(price_data.timestamp_, 3); - assert_eq!(price_data.prev_slot_, 1000); - assert_eq!(price_data.prev_price_, 145); - assert_eq!(price_data.prev_conf_, 55); - assert_eq!(price_data.prev_timestamp_, 2); } // four publishers @@ -226,10 +218,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.num_qt_, 4); assert_eq!(price_data.timestamp_, 4); assert_eq!(price_data.last_slot_, 1001); - assert_eq!(price_data.prev_slot_, 1000); - assert_eq!(price_data.prev_price_, 200); - assert_eq!(price_data.prev_conf_, 90); - assert_eq!(price_data.prev_timestamp_, 3); } unsafe { @@ -247,10 +235,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.last_slot_, 1025); assert_eq!(price_data.num_qt_, 4); assert_eq!(price_data.timestamp_, 5); - assert_eq!(price_data.prev_slot_, 1001); - assert_eq!(price_data.prev_price_, 245); - assert_eq!(price_data.prev_conf_, 85); - assert_eq!(price_data.prev_timestamp_, 4); } // check what happens when nothing publishes for a while @@ -269,10 +253,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.last_slot_, 1025); assert_eq!(price_data.num_qt_, 0); assert_eq!(price_data.timestamp_, 10); - assert_eq!(price_data.prev_slot_, 1025); - assert_eq!(price_data.prev_price_, 245); - assert_eq!(price_data.prev_conf_, 85); - assert_eq!(price_data.prev_timestamp_, 5); } unsafe { @@ -290,10 +270,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.last_slot_, 1025); assert_eq!(price_data.num_qt_, 0); assert_eq!(price_data.timestamp_, 12); - assert_eq!(price_data.prev_slot_, 1025); - assert_eq!(price_data.prev_price_, 245); - assert_eq!(price_data.prev_conf_, 85); - assert_eq!(price_data.prev_timestamp_, 5); } // ensure the update occurs within the PC_MAX_SEND_LATENCY limit of 25 slots, allowing the aggregated price to reflect both p4 and p5 contributions @@ -322,10 +298,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.conf_, 55); assert_eq!(price_data.num_qt_, 2); assert_eq!(price_data.timestamp_, 13); - assert_eq!(price_data.prev_slot_, 1025); - assert_eq!(price_data.prev_price_, 245); - assert_eq!(price_data.prev_conf_, 85); - assert_eq!(price_data.prev_timestamp_, 5); } // verify behavior when publishing halts for 1 slot, causing the slot difference from p5 to exceed the PC_MAX_SEND_LATENCY threshold of 25. @@ -345,10 +317,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.conf_, 50); assert_eq!(price_data.num_qt_, 1); assert_eq!(price_data.timestamp_, 14); - assert_eq!(price_data.prev_slot_, 1025); - assert_eq!(price_data.prev_price_, 445); - assert_eq!(price_data.prev_conf_, 55); - assert_eq!(price_data.prev_timestamp_, 13); } // verify behavior when max_latency_ is set to 5, and all components pub_slot_ gap is more than 5, this should result in PC_STATUS_UNKNOWN status @@ -380,10 +348,6 @@ fn test_upd_aggregate() { assert_eq!(price_data.agg_.conf_, 50); assert_eq!(price_data.num_qt_, 0); assert_eq!(price_data.timestamp_, 15); - assert_eq!(price_data.prev_slot_, 1000); - assert_eq!(price_data.prev_price_, 500); - assert_eq!(price_data.prev_conf_, 50); - assert_eq!(price_data.prev_timestamp_, 14); } } diff --git a/program/rust/src/tests/test_upd_price_v2.rs b/program/rust/src/tests/test_upd_price_v2.rs index daaa0f88..0af7ee1e 100644 --- a/program/rust/src/tests/test_upd_price_v2.rs +++ b/program/rust/src/tests/test_upd_price_v2.rs @@ -480,10 +480,13 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.agg_.price_, 14); assert_eq!(price_data.agg_.conf_, 6); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 51); + assert_eq!(price_data.prev_price_, 55); + assert_eq!(price_data.prev_conf_, 5); assert_eq!( price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 + 14 * 49 - ); + ); // (42 + 81 * 4 - 100 * 4 + 60 * 41 + 55) is the price cumulative from the previous test and 14 * 49 (slot_gap) is the price cumulative from this test assert_eq!( price_data.price_cumulative.conf, 2 * 5 + 4 + 41 * 4 + 5 + 6 * 49 From e4c585e7024e2bf807af5cdf57685d71b7c51722 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 22:54:57 +0900 Subject: [PATCH 21/60] add ema test --- program/rust/src/tests/test_ema.rs | 126 +++++++++++++++++++- program/rust/src/tests/test_upd_price_v2.rs | 9 ++ 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index 73b15f2c..22864c83 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -1,12 +1,30 @@ extern crate test_generator; use { + super::test_utils::update_clock_slot, crate::{ - accounts::PriceAccount, + accounts::{ + PriceAccount, + PythAccount, + }, + c_oracle_header::{ + PC_STATUS_TRADING, + PC_VERSION, + }, + deserialize::{ + load_checked, + load_mut, + }, + instruction::{ + OracleCommand, + UpdPriceArgs, + }, processor::{ c_upd_aggregate, c_upd_twap, + process_instruction, }, + tests::test_utils::AccountSetup, }, bytemuck::Zeroable, csv::ReaderBuilder, @@ -14,10 +32,104 @@ use { Deserialize, Serialize, }, - std::fs::File, + solana_program::pubkey::Pubkey, + std::{ + fs::File, + mem::size_of, + }, test_generator::test_resources, }; +#[test] +fn test_ema_multiple_publishers_same_slot() -> Result<(), Box> { + let mut instruction_data = [0u8; size_of::()]; + populate_instruction(&mut instruction_data, 10, 1, 1); + + let program_id = Pubkey::new_unique(); + + let mut funding_setup = AccountSetup::new_funding(); + let funding_account = funding_setup.as_account_info(); + + let mut price_setup = AccountSetup::new::(&program_id); + let mut price_account = price_setup.as_account_info(); + price_account.is_signer = false; + PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); + + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.num_ = 1; + price_data.comp_[0].pub_ = *funding_account.key; + } + + let mut clock_setup = AccountSetup::new_clock(); + let mut clock_account = clock_setup.as_account_info(); + clock_account.is_signer = false; + clock_account.is_writable = false; + + update_clock_slot(&mut clock_account, 1); + + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.prev_twap_.val_, 0); + assert_eq!(price_data.prev_twac_.val_, 0); + assert_eq!(price_data.twap_.val_, 10); + assert_eq!(price_data.twac_.val_, 1); + } + + // add new test for multiple publishers and ensure that price_cumulative is updated correctly + let mut funding_setup_two = AccountSetup::new_funding(); + let funding_account_two = funding_setup_two.as_account_info(); + + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.num_ = 2; + price_data.comp_[1].pub_ = *funding_account_two.key; + } + + populate_instruction(&mut instruction_data, 20, 2, 2); + update_clock_slot(&mut clock_account, 2); + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + populate_instruction(&mut instruction_data, 30, 3, 2); + process_instruction( + &program_id, + &[ + funding_account_two.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.agg_.price_, 24); + assert_eq!(price_data.prev_twap_.val_, 10); + assert_eq!(price_data.prev_twac_.val_, 1); + assert_eq!(price_data.twap_.val_, 11); + assert_eq!(price_data.twac_.val_, 1); + } + Ok(()) +} + #[test_resources("program/rust/test_data/ema/*.csv")] fn test_ema(input_path_raw: &str) { let (inputs, expected_outputs) = read_test_data(input_path_raw); @@ -145,3 +257,13 @@ struct OutputRecord { twap: i64, twac: i64, } + +fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_slot: u64) { + let mut cmd = load_mut::(instruction_data).unwrap(); + cmd.header = OracleCommand::UpdPrice.into(); + cmd.status = PC_STATUS_TRADING; + cmd.price = price; + cmd.confidence = conf; + cmd.publishing_slot = pub_slot; + cmd.unused_ = 0; +} diff --git a/program/rust/src/tests/test_upd_price_v2.rs b/program/rust/src/tests/test_upd_price_v2.rs index 0af7ee1e..a2e326a0 100644 --- a/program/rust/src/tests/test_upd_price_v2.rs +++ b/program/rust/src/tests/test_upd_price_v2.rs @@ -483,6 +483,15 @@ fn test_upd_price_v2() -> Result<(), Box> { assert_eq!(price_data.prev_slot_, 51); assert_eq!(price_data.prev_price_, 55); assert_eq!(price_data.prev_conf_, 5); + assert_eq!( + price_data.prev_price_cumulative.price, + 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 + ); + assert_eq!( + price_data.prev_price_cumulative.conf, + 2 * 5 + 4 + 41 * 4 + 5 + ); + assert_eq!(price_data.prev_price_cumulative.num_down_slots, 16); assert_eq!( price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 + 14 * 49 From 6bb795c5b4fead801a7cbaf7e7c0acd62b529453 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 20 Mar 2024 23:14:38 +0900 Subject: [PATCH 22/60] reduce PC_NUM_COMP_PYTHNET to 64 --- program/c/src/oracle/oracle.h | 2 +- program/rust/src/accounts/price.rs | 54 ++++------------------------ program/rust/src/tests/test_sizes.rs | 6 ++++ 3 files changed, 14 insertions(+), 48 deletions(-) diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index d887d5b6..78eec458 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -24,7 +24,7 @@ extern "C" { // Total price component slots available #define PC_NUM_COMP_SOLANA 32 -#define PC_NUM_COMP_PYTHNET 127 +#define PC_NUM_COMP_PYTHNET 64 // PC_NUM_COMP - number of price components in use #ifdef PC_PYTHNET diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index f0fa23a7..e46ee20b 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -35,54 +35,8 @@ mod price_pythnet { }, error::OracleError, }, - std::ops::{ - Deref, - DerefMut, - Index, - IndexMut, - }, }; - #[repr(C)] - #[derive(Copy, Clone)] - pub struct PriceComponentArrayWrapper([PriceComponent; PC_NUM_COMP_PYTHNET as usize]); - - // Implementing Index and IndexMut allows PriceComponentArrayWrapper to use array indexing directly, - // such as price_account.comp_[i], making it behave more like a native array or slice. - impl Index for PriceComponentArrayWrapper { - type Output = PriceComponent; - - fn index(&self, index: usize) -> &Self::Output { - &self.0[index] - } - } - impl IndexMut for PriceComponentArrayWrapper { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.0[index] - } - } - - // Implementing Deref and DerefMut allows PriceComponentArrayWrapper to use slice methods directly, - // such as len(), making it behave more like a native array or slice. - impl Deref for PriceComponentArrayWrapper { - type Target = [PriceComponent]; - - fn deref(&self) -> &Self::Target { - &self.0 - } - } - - impl DerefMut for PriceComponentArrayWrapper { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } - } - - unsafe impl Pod for PriceComponentArrayWrapper { - } - unsafe impl Zeroable for PriceComponentArrayWrapper { - } - /// Pythnet-only extended price account format. This extension is /// an append-only change that adds extra publisher slots and /// PriceCumulative for TWAP processing. @@ -134,7 +88,13 @@ mod price_pythnet { /// PC_NUM_COMP_PYTHNET slots are used due to stack size /// issues in the C code. For iterating over price components, /// PC_NUM_COMP must be used. - pub comp_: PriceComponentArrayWrapper, + pub comp_: [PriceComponent; PC_NUM_COMP_PYTHNET as usize], + pub unused_: [u8; 4096], + pub unused2_: [u8; 1024], + pub unused3_: [u8; 512], + pub unused4_: [u8; 256], + pub unused5_: [u8; 128], + pub unused6_: [u8; 32], /// Previous EMA for price and confidence pub prev_twap_: PriceEma, pub prev_twac_: PriceEma, diff --git a/program/rust/src/tests/test_sizes.rs b/program/rust/src/tests/test_sizes.rs index b3ec7507..e8688595 100644 --- a/program/rust/src/tests/test_sizes.rs +++ b/program/rust/src/tests/test_sizes.rs @@ -69,6 +69,12 @@ fn test_sizes() { + 3 * size_of::() + size_of::() + (PC_NUM_COMP_PYTHNET as usize) * size_of::() + + 4096 // Size of unused_ + + 1024 // Size of unused2_ + + 512 // Size of unused3_ + + 256 // Size of unused4_ + + 128 // Size of unused5_ + + 32 // Size of unused6_ + size_of::() + size_of::() + size_of::() From 18912de6a54b8f5d2f9e5b7537be12507ff2ca14 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 21 Mar 2024 11:57:01 +0900 Subject: [PATCH 23/60] fix tests --- program/rust/src/processor/upd_price.rs | 8 +++++++- program/rust/src/tests/test_ema.rs | 7 +++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 180d63ec..37d6bc2c 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -252,9 +252,15 @@ pub fn upd_price( price_data.update_price_cumulative()?; } unsafe { + let prev_slot = { + // get price data here + let new_price_data = + load_checked::(price_account, cmd_args.header.version)?; + new_price_data.prev_slot_ + }; c_upd_twap( price_account.try_borrow_mut_data()?.as_mut_ptr(), - slots_since_last_update as i64, // Ensure slots_since_last_update is cast to i64, as expected by the function signature + (clock.slot - prev_slot) as i64, // Ensure slots_since_last_update is cast to i64, as expected by the function signature ); } } diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index 22864c83..4f22370d 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -96,7 +96,7 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box Result<(), Box Result<(), Box(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.agg_.price_, 24); assert_eq!(price_data.prev_twap_.val_, 10); assert_eq!(price_data.prev_twac_.val_, 1); - assert_eq!(price_data.twap_.val_, 11); + assert_eq!(price_data.twap_.val_, 12); assert_eq!(price_data.twac_.val_, 1); } Ok(()) From 972586931d5973495f93b0c7c79af4714cefe891 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 21 Mar 2024 11:59:08 +0900 Subject: [PATCH 24/60] add comments --- program/rust/src/processor/upd_price.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 37d6bc2c..7a4b5599 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -203,7 +203,7 @@ pub fn upd_price( } let should_call_c_upd_aggregate: bool = if cfg!(feature = "pythnet") { - clock.slot >= latest_aggregate_price.pub_slot_ + true // clock.slot >= latest_aggregate_price.pub_slot_ is always true } else { clock.slot > latest_aggregate_price.pub_slot_ }; From 26a07d585051739f4afedcf0be8ab1b57b2b2bfa Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 21 Mar 2024 14:01:06 +0900 Subject: [PATCH 25/60] revert to use PriceComponentArrayWrapper --- program/c/src/oracle/oracle.h | 2 +- program/rust/src/accounts/price.rs | 54 ++++++++++++++++++++++++---- program/rust/src/tests/test_sizes.rs | 6 ---- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index 78eec458..d887d5b6 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -24,7 +24,7 @@ extern "C" { // Total price component slots available #define PC_NUM_COMP_SOLANA 32 -#define PC_NUM_COMP_PYTHNET 64 +#define PC_NUM_COMP_PYTHNET 127 // PC_NUM_COMP - number of price components in use #ifdef PC_PYTHNET diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index e46ee20b..f0fa23a7 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -35,8 +35,54 @@ mod price_pythnet { }, error::OracleError, }, + std::ops::{ + Deref, + DerefMut, + Index, + IndexMut, + }, }; + #[repr(C)] + #[derive(Copy, Clone)] + pub struct PriceComponentArrayWrapper([PriceComponent; PC_NUM_COMP_PYTHNET as usize]); + + // Implementing Index and IndexMut allows PriceComponentArrayWrapper to use array indexing directly, + // such as price_account.comp_[i], making it behave more like a native array or slice. + impl Index for PriceComponentArrayWrapper { + type Output = PriceComponent; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } + } + impl IndexMut for PriceComponentArrayWrapper { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } + } + + // Implementing Deref and DerefMut allows PriceComponentArrayWrapper to use slice methods directly, + // such as len(), making it behave more like a native array or slice. + impl Deref for PriceComponentArrayWrapper { + type Target = [PriceComponent]; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl DerefMut for PriceComponentArrayWrapper { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } + } + + unsafe impl Pod for PriceComponentArrayWrapper { + } + unsafe impl Zeroable for PriceComponentArrayWrapper { + } + /// Pythnet-only extended price account format. This extension is /// an append-only change that adds extra publisher slots and /// PriceCumulative for TWAP processing. @@ -88,13 +134,7 @@ mod price_pythnet { /// PC_NUM_COMP_PYTHNET slots are used due to stack size /// issues in the C code. For iterating over price components, /// PC_NUM_COMP must be used. - pub comp_: [PriceComponent; PC_NUM_COMP_PYTHNET as usize], - pub unused_: [u8; 4096], - pub unused2_: [u8; 1024], - pub unused3_: [u8; 512], - pub unused4_: [u8; 256], - pub unused5_: [u8; 128], - pub unused6_: [u8; 32], + pub comp_: PriceComponentArrayWrapper, /// Previous EMA for price and confidence pub prev_twap_: PriceEma, pub prev_twac_: PriceEma, diff --git a/program/rust/src/tests/test_sizes.rs b/program/rust/src/tests/test_sizes.rs index e8688595..b3ec7507 100644 --- a/program/rust/src/tests/test_sizes.rs +++ b/program/rust/src/tests/test_sizes.rs @@ -69,12 +69,6 @@ fn test_sizes() { + 3 * size_of::() + size_of::() + (PC_NUM_COMP_PYTHNET as usize) * size_of::() - + 4096 // Size of unused_ - + 1024 // Size of unused2_ - + 512 // Size of unused3_ - + 256 // Size of unused4_ - + 128 // Size of unused5_ - + 32 // Size of unused6_ + size_of::() + size_of::() + size_of::() From 42aadde1a3f0bf09c6e4c12f22fc03d22a09ed79 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 13:30:18 +0900 Subject: [PATCH 26/60] refactor --- .github/workflows/docker.yaml | 33 +- .pre-commit-config.yaml | 45 +- program/c/makefile | 9 +- program/c/src/oracle/native/upd_aggregate.c | 4 - program/c/src/oracle/oracle.h | 843 +++++++++--------- program/c/src/oracle/upd_aggregate.c | 6 +- program/c/src/oracle/upd_aggregate.h | 389 ++++---- program/rust/Cargo.toml | 2 - program/rust/build.rs | 18 +- program/rust/src/accounts.rs | 4 +- program/rust/src/accounts/price.rs | 77 -- program/rust/src/processor/upd_price.rs | 172 ++-- program/rust/src/tests/mod.rs | 2 - program/rust/src/tests/pyth_simulator.rs | 42 +- program/rust/src/tests/test_publish.rs | 42 +- program/rust/src/tests/test_publish_batch.rs | 58 +- program/rust/src/tests/test_sizes.rs | 64 +- program/rust/src/tests/test_upd_price.rs | 71 -- .../tests/test_upd_price_no_fail_on_error.rs | 24 +- scripts/build-bpf.sh | 13 +- 20 files changed, 770 insertions(+), 1148 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 06b57dc4..9c5c437a 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -2,11 +2,11 @@ name: Docker on: push: - branches: [ main ] - tags : + branches: [main] + tags: - oracle-v* pull_request: - branches: [ main ] + branches: [main] env: SOLANA_VERSION: 1.14.7 # Using 1.14.x to have sbf instead of bpf. bpf does not work anymore. @@ -19,7 +19,6 @@ env: IS_ORACLE_RELEASE: ${{ startsWith( github.ref, 'refs/tags/oracle-' ) }} - jobs: build: runs-on: ubuntu-latest @@ -56,25 +55,15 @@ jobs: } echo "${{ secrets.DOCKER_IO_PASS }}" | publish - - name : Get binary from docker - if : env.IS_ORACLE_RELEASE == 'true' - run : | - docker create -ti --name container "${DOCKER_IMAGE}" bash - docker cp container:/home/pyth/pyth-client/target/pyth/pythnet/pyth_oracle_pythnet.so . - docker cp container:/home/pyth/pyth-client/target/pyth/solana/pyth_oracle_solana.so . - docker rm -f container + - name: Get binary from docker + if: env.IS_ORACLE_RELEASE == 'true' + run: | + docker create -ti --name container "${DOCKER_IMAGE}" bash + docker cp container:/home/pyth/pyth-client/target/pyth/pythnet/pyth_oracle_pythnet.so . + docker rm -f container - - name : Publish Solana binary - if : env.IS_ORACLE_RELEASE == 'true' - uses: svenstaro/upload-release-action@133984371c30d34e38222a64855679a414cb7575 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ./pyth_oracle_solana.so - asset_name: pyth_oracle_solana.so - tag: ${{ github.ref }} - - - name : Publish Pythnet binary - if : env.IS_ORACLE_RELEASE == 'true' + - name: Publish Pythnet binary + if: env.IS_ORACLE_RELEASE == 'true' uses: svenstaro/upload-release-action@133984371c30d34e38222a64855679a414cb7575 with: repo_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbb67811..867aa05f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,26 +1,21 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: trailing-whitespace - files : program/ - - id: end-of-file-fixer - files : program/ - - id: check-added-large-files -- repo: local - hooks: - - id: cargo-fmt - name: Cargo Fmt - language: "rust" - entry: cargo +nightly-2023-03-01 fmt - pass_filenames: false - - id: cargo-clippy-solana - name: Cargo Clippy Solana - language: "rust" - entry : cargo +nightly-2023-03-01 clippy --tests --features check -- -D warnings - pass_filenames : false - - id: cargo-clippy-pythnet - name: Cargo Clippy Pythnet - language: "rust" - entry : cargo +nightly-2023-03-01 clippy --tests --features pythnet,check -- -D warnings - pass_filenames : false + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + files: program/ + - id: end-of-file-fixer + files: program/ + - id: check-added-large-files + - repo: local + hooks: + - id: cargo-fmt + name: Cargo Fmt + language: "rust" + entry: cargo +nightly-2023-03-01 fmt + pass_filenames: false + - id: cargo-clippy-pythnet + name: Cargo Clippy Pythnet + language: "rust" + entry: cargo +nightly-2023-03-01 clippy --tests --features check -- -D warnings + pass_filenames: false \ No newline at end of file diff --git a/program/c/makefile b/program/c/makefile index 563661b1..3b791858 100644 --- a/program/c/makefile +++ b/program/c/makefile @@ -11,14 +11,7 @@ else include $(SOLANA)/sdk/sbf/c/sbf.mk endif -# Propagate the PC_PYTHNET feature by conditionally defining it in a -# features.h header. The makefile included from Solana SDK does not -# have an easy way to pass extra C flags which motivates this approach. -ifdef PC_PYTHNET - FEATURES_H_BODY:="\#pragma once\n\#define PC_PYTHNET 1" -else - FEATURES_H_BODY:="\#pragma once" -endif +FEATURES_H_BODY:="\#pragma once" .PHONY: features.h # Putting this in .PHONY makes sure the header is always regenerated diff --git a/program/c/src/oracle/native/upd_aggregate.c b/program/c/src/oracle/native/upd_aggregate.c index 7c300685..ef6220d8 100644 --- a/program/c/src/oracle/native/upd_aggregate.c +++ b/program/c/src/oracle/native/upd_aggregate.c @@ -9,11 +9,7 @@ char heap_start[8192]; #include "../upd_aggregate.h" #include "../features.h" -#ifdef PC_PYTHNET extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#else -extern bool c_upd_aggregate_solana( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#endif return upd_aggregate(ptr, slot, timestamp ); } diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index d887d5b6..5e108e9f 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -5,454 +5,469 @@ #include "features.h" #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif // magic number at head of account #define PC_MAGIC 0xa1b2c3d4 // current program version -#define PC_VERSION 2 +#define PC_VERSION 2 // max latency in slots between send and receive -#define PC_MAX_SEND_LATENCY 25 +#define PC_MAX_SEND_LATENCY 25 // various size constants -#define PC_PUBKEY_SIZE 32 -#define PC_PUBKEY_SIZE_64 (PC_PUBKEY_SIZE/sizeof(uint64_t)) -#define PC_MAP_TABLE_SIZE 640 +#define PC_PUBKEY_SIZE 32 +#define PC_PUBKEY_SIZE_64 (PC_PUBKEY_SIZE / sizeof(uint64_t)) +#define PC_MAP_TABLE_SIZE 640 -// Total price component slots available -#define PC_NUM_COMP_SOLANA 32 -#define PC_NUM_COMP_PYTHNET 127 + // Total price component slots available +#define PC_NUM_COMP_PYTHNET 127 // PC_NUM_COMP - number of price components in use -#ifdef PC_PYTHNET // Not whole PC_NUM_COMP_PYTHNET because of stack issues appearing in upd_aggregate() #define PC_NUM_COMP 64 -#else -#define PC_NUM_COMP PC_NUM_COMP_SOLANA -#endif -#define PC_PROD_ACC_SIZE 512 -#define PC_EXP_DECAY -9 +#define PC_PROD_ACC_SIZE 512 +#define PC_EXP_DECAY -9 #ifndef PC_HEAP_START #define PC_HEAP_START (0x300000000) #endif // price types -#define PC_PTYPE_UNKNOWN 0 -#define PC_PTYPE_PRICE 1 +#define PC_PTYPE_UNKNOWN 0 +#define PC_PTYPE_PRICE 1 // symbol status -#define PC_STATUS_UNKNOWN 0 -#define PC_STATUS_TRADING 1 -#define PC_STATUS_HALTED 2 -#define PC_STATUS_AUCTION 3 -#define PC_STATUS_IGNORED 4 +#define PC_STATUS_UNKNOWN 0 +#define PC_STATUS_TRADING 1 +#define PC_STATUS_HALTED 2 +#define PC_STATUS_AUCTION 3 +#define PC_STATUS_IGNORED 4 // account types -#define PC_ACCTYPE_MAPPING 1 -#define PC_ACCTYPE_PRODUCT 2 -#define PC_ACCTYPE_PRICE 3 -#define PC_ACCTYPE_TEST 4 -#define PC_ACCTYPE_PERMISSIONS 5 - +#define PC_ACCTYPE_MAPPING 1 +#define PC_ACCTYPE_PRODUCT 2 +#define PC_ACCTYPE_PRICE 3 +#define PC_ACCTYPE_TEST 4 +#define PC_ACCTYPE_PERMISSIONS 5 // Compute budget requested per price update instruction // The biggest instruction appears to be about ~10300 CUs, so overestimate by 100%. #define CU_BUDGET_PER_IX 20000 -// binary version of sysvar_clock account id -const uint64_t sysvar_clock[] = { - 0xc974c71817d5a706UL, - 0xb65e1d6998635628UL, - 0x5c6d4b9ba3b85e8bUL, - 0x215b5573UL -}; + // binary version of sysvar_clock account id + const uint64_t sysvar_clock[] = { + 0xc974c71817d5a706UL, + 0xb65e1d6998635628UL, + 0x5c6d4b9ba3b85e8bUL, + 0x215b5573UL}; + + // compute budget program id in hex (but wrong endianness) + /* + 321721e56f460603 + e79bc372baadecff + 6b12f7c5bbe58cbc + 403a9b432c + */ + + // correct + const uint64_t compute_budget_program_id[] = { + 0x321721e56f460603UL, + 0xe79bc372baadecffUL, + 0x6b12f7c5bbe58cbcUL, + 0x403a9b432cUL}; + + // public key of symbol or publisher account + typedef union pc_pub_key + { + uint8_t k1_[PC_PUBKEY_SIZE]; + uint64_t k8_[PC_PUBKEY_SIZE_64]; + } pc_pub_key_t; + + static_assert(sizeof(pc_pub_key_t) == 32, ""); + + // account header information + typedef struct pc_acc + { + uint32_t magic_; // pyth magic number + uint32_t ver_; // program/account version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + } pc_acc_t; + + static_assert(sizeof(pc_acc_t) == 16, ""); + + // hash table of symbol to price account mappings + typedef struct pc_map_table + { + uint32_t magic_; // pyth magic number + uint32_t ver_; // program/account version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + uint32_t num_; // number of symbols + uint32_t unused_; // 64bit padding + pc_pub_key_t next_; // next mapping account in chain + pc_pub_key_t prod_[PC_MAP_TABLE_SIZE]; // product accounts + } pc_map_table_t; + + static_assert(sizeof(pc_map_table_t) == 20536, ""); + + // variable length string + typedef struct pc_str + { + uint8_t len_; + char data_[]; + } pc_str_t; + + static_assert(sizeof(pc_str_t) == 1, ""); + + // product reference data + typedef struct pc_prod + { + uint32_t magic_; + uint32_t ver_; // program version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + pc_pub_key_t px_acc_; // first price (or quote) account + // variable number of reference key/value attribute pairs + // stored as strings (pc_str_t) + } pc_prod_t; + + static_assert(sizeof(pc_prod_t) == 48, ""); + + // price et al. for some component or aggregate + typedef struct pc_price_info + { + int64_t price_; // price per ptype_ + uint64_t conf_; // price confidence interval + uint32_t status_; // symbol status as of last update + uint32_t corp_act_status_; // corp action status as of last update + uint64_t pub_slot_; // publish slot of price + } pc_price_info_t; + + static_assert(sizeof(pc_price_info_t) == 32, ""); + + // published component price for contributing provider + typedef struct pc_price_comp + { + pc_pub_key_t pub_; // publishing key of component price + pc_price_info_t agg_; // price used in aggregate calc + pc_price_info_t latest_; // latest contributed prices + } pc_price_comp_t; + + static_assert(sizeof(pc_price_comp_t) == 96, ""); + + // time-weighted exponential moving average + typedef struct pc_ema + { + int64_t val_; // current value of ema + int64_t numer_; // numerator at full precision + int64_t denom_; // denominator at full precision + } pc_ema_t; + + static_assert(sizeof(pc_ema_t) == 24, ""); + + // price account containing aggregate and all component prices + typedef struct pc_price + { + uint32_t magic_; // pyth magic number + uint32_t ver_; // program version + uint32_t type_; // account type + uint32_t size_; // price account size + uint32_t ptype_; // price or calculation type + int32_t expo_; // price exponent + uint32_t num_; // number of component prices + uint32_t num_qt_; // number of quoters that make up aggregate + uint64_t last_slot_; // slot of last valid aggregate price + uint64_t valid_slot_; // valid on-chain slot of agg. price + pc_ema_t twap_; // time-weighted average price + pc_ema_t twac_; // time-weighted average conf interval + int64_t timestamp_; // unix timestamp of aggregate price + uint8_t min_pub_; // min publishers for valid price + int8_t message_sent_; // flag to indicate if the current aggregate price has been sent as a message to the message buffer, 0 if not sent, 1 if sent + uint8_t max_latency_; // configurable max latency in slots between send and receive + int8_t drv3_; // space for future derived values + int32_t drv4_; // space for future derived values + pc_pub_key_t prod_; // product id/ref-account + pc_pub_key_t next_; // next price account in list + uint64_t prev_slot_; // valid slot of previous aggregate with TRADING status + int64_t prev_price_; // aggregate price of previous aggregate with TRADING status + uint64_t prev_conf_; // confidence interval of previous aggregate with TRADING status + int64_t prev_timestamp_; // unix timestamp of previous aggregate with TRADING status + pc_price_info_t agg_; // aggregate price information + pc_price_comp_t comp_[PC_NUM_COMP]; // component prices + } pc_price_t; -// compute budget program id in hex (but wrong endianness) /* -321721e56f460603 -e79bc372baadecff -6b12f7c5bbe58cbc -403a9b432c +The value 240 is derived from the fixed size of the pc_price_t struct excluding the size of the comp_ array. +Here is the breakdown of the sizes (in bytes) for each field within the pc_price_t struct: + +- magic_ (uint32_t): 4 bytes +- ver_ (uint32_t): 4 bytes +- type_ (uint32_t): 4 bytes +- size_ (uint32_t): 4 bytes +- ptype_ (uint32_t): 4 bytes +- expo_ (int32_t): 4 bytes +- num_ (uint32_t): 4 bytes +- num_qt_ (uint32_t): 4 bytes +- last_slot_ (uint64_t): 8 bytes +- valid_slot_ (uint64_t): 8 bytes +- twap_ (pc_ema_t): 24 bytes (3 fields of int64_t each taking 8 bytes) +- twac_ (pc_ema_t): 24 bytes (similar to twap_) +- timestamp_ (int64_t): 8 bytes +- min_pub_ (uint8_t): 1 byte +- message_sent_ (int8_t): 1 byte +- max_latency_ (uint8_t): 1 byte +- drv3_ (int8_t): 1 byte +- drv4_ (int32_t): 4 bytes +- prod_ (pc_pub_key_t): 32 bytes (assuming pc_pub_key_t is a 32-byte array or struct) +- next_ (pc_pub_key_t): 32 bytes (similar to prod_) +- prev_slot_ (uint64_t): 8 bytes +- prev_price_ (int64_t): 8 bytes +- prev_conf_ (uint64_t): 8 bytes +- prev_timestamp_ (int64_t): 8 bytes +- agg_ (pc_price_info_t): 32 bytes + +Adding up all these sizes gives us a total of 240 bytes for the fixed part of the pc_price_t struct. +The size of the comp_ array is variable and depends on PC_NUM_COMP, hence it is calculated separately and added to the base size of 240 bytes. */ - -// correct -const uint64_t compute_budget_program_id[] = { - 0x321721e56f460603UL, - 0xe79bc372baadecffUL, - 0x6b12f7c5bbe58cbcUL, - 0x403a9b432cUL -}; - -// public key of symbol or publisher account -typedef union pc_pub_key -{ - uint8_t k1_[PC_PUBKEY_SIZE]; - uint64_t k8_[PC_PUBKEY_SIZE_64]; -} pc_pub_key_t; - -static_assert( sizeof( pc_pub_key_t ) == 32, "" ); - -// account header information -typedef struct pc_acc -{ - uint32_t magic_; // pyth magic number - uint32_t ver_; // program/account version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account -} pc_acc_t; - -static_assert( sizeof( pc_acc_t ) == 16, "" ); - -// hash table of symbol to price account mappings -typedef struct pc_map_table -{ - uint32_t magic_; // pyth magic number - uint32_t ver_; // program/account version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - uint32_t num_; // number of symbols - uint32_t unused_; // 64bit padding - pc_pub_key_t next_; // next mapping account in chain - pc_pub_key_t prod_[PC_MAP_TABLE_SIZE]; // product accounts -} pc_map_table_t; - -static_assert( sizeof( pc_map_table_t ) == 20536, "" ); - -// variable length string -typedef struct pc_str -{ - uint8_t len_; - char data_[]; -} pc_str_t; - -static_assert( sizeof( pc_str_t ) == 1, "" ); - -// product reference data -typedef struct pc_prod -{ - uint32_t magic_; - uint32_t ver_; // program version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - pc_pub_key_t px_acc_; // first price (or quote) account - // variable number of reference key/value attribute pairs - // stored as strings (pc_str_t) -} pc_prod_t; - -static_assert( sizeof( pc_prod_t ) == 48, "" ); - -// price et al. for some component or aggregate -typedef struct pc_price_info -{ - int64_t price_; // price per ptype_ - uint64_t conf_; // price confidence interval - uint32_t status_; // symbol status as of last update - uint32_t corp_act_status_; // corp action status as of last update - uint64_t pub_slot_; // publish slot of price -} pc_price_info_t; - -static_assert( sizeof( pc_price_info_t ) == 32, "" ); - -// published component price for contributing provider -typedef struct pc_price_comp -{ - pc_pub_key_t pub_; // publishing key of component price - pc_price_info_t agg_; // price used in aggregate calc - pc_price_info_t latest_; // latest contributed prices -} pc_price_comp_t; - -static_assert( sizeof( pc_price_comp_t ) == 96, "" ); - -// time-weighted exponential moving average -typedef struct pc_ema -{ - int64_t val_; // current value of ema - int64_t numer_; // numerator at full precision - int64_t denom_; // denominator at full precision -} pc_ema_t; - -static_assert( sizeof( pc_ema_t ) == 24, "" ); - -// price account containing aggregate and all component prices -typedef struct pc_price -{ - uint32_t magic_; // pyth magic number - uint32_t ver_; // program version - uint32_t type_; // account type - uint32_t size_; // price account size - uint32_t ptype_; // price or calculation type - int32_t expo_; // price exponent - uint32_t num_; // number of component prices - uint32_t num_qt_; // number of quoters that make up aggregate - uint64_t last_slot_; // slot of last valid aggregate price - uint64_t valid_slot_; // valid on-chain slot of agg. price - pc_ema_t twap_; // time-weighted average price - pc_ema_t twac_; // time-weighted average conf interval - int64_t timestamp_; // unix timestamp of aggregate price - uint8_t min_pub_; // min publishers for valid price - int8_t message_sent_; // flag to indicate if the current aggregate price has been sent as a message to the message buffer, 0 if not sent, 1 if sent - uint8_t max_latency_; // configurable max latency in slots between send and receive - int8_t drv3_; // space for future derived values - int32_t drv4_; // space for future derived values - pc_pub_key_t prod_; // product id/ref-account - pc_pub_key_t next_; // next price account in list - uint64_t prev_slot_; // valid slot of previous aggregate with TRADING status - int64_t prev_price_; // aggregate price of previous aggregate with TRADING status - uint64_t prev_conf_; // confidence interval of previous aggregate with TRADING status - int64_t prev_timestamp_; // unix timestamp of previous aggregate with TRADING status - pc_price_info_t agg_; // aggregate price information - pc_price_comp_t comp_[PC_NUM_COMP];// component prices -} pc_price_t; - -#ifdef PC_PYTHNET - -// Replace Solana component size's contribution with Pythnet's -#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (3312 \ - - PC_NUM_COMP_SOLANA * sizeof(pc_price_comp_t) \ - + PC_NUM_COMP * sizeof(pc_price_comp_t) \ - ) - -static_assert( sizeof( pc_price_t ) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, "" ); -#undef PC_EXPECTED_PRICE_SIZE_PYTHNET - -#else -static_assert( sizeof( pc_price_t ) == 3312, "" ); -#endif - -// This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd. -// It is set tighly to the current price account + 96 component prices + 48 bytes for cumulative sums -const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof( pc_price_comp_t) + 48; - - -// command enumeration -typedef enum { - - // initialize first mapping list account - // key[0] funding account [signer writable] - // key[1] mapping account [signer writable] - e_cmd_init_mapping = 0, - - // initialize and add new mapping account - // key[0] funding account [signer writable] - // key[1] tail mapping account [signer writable] - // key[2] new mapping account [signer writable] - e_cmd_add_mapping, - - // initialize and add new product reference data account - // key[0] funding account [signer writable] - // key[1] mapping account [signer writable] - // key[2] new product account [signer writable] - e_cmd_add_product, - - // update product account - // key[0] funding account [signer writable] - // key[1] product account [signer writable] - e_cmd_upd_product, - - // add new price account to a product account - // key[0] funding account [signer writable] - // key[1] product account [signer writable] - // key[2] new price account [signer writable] - e_cmd_add_price, - - // add publisher to symbol account - // key[0] funding account [signer writable] - // key[1] price account [signer writable] - e_cmd_add_publisher, - - // delete publisher from symbol account - // key[0] funding account [signer writable] - // key[1] price account [signer writable] - e_cmd_del_publisher, - - // publish component price - // key[0] funding account [signer writable] - // key[1] price account [writable] - // key[2] sysvar_clock account [readable] - e_cmd_upd_price, - - // compute aggregate price - // key[0] funding account [signer writable] - // key[1] price account [writable] - // key[2] sysvar_clock account [readable] - e_cmd_agg_price, - - // (re)initialize price account - // key[0] funding account [signer writable] - // key[1] new price account [signer writable] - e_cmd_init_price, - - // deprecated - e_cmd_init_test, - - // deprecated - e_cmd_upd_test, - - // set min publishers - // key[0] funding account [signer writable] - // key[1] price account [signer writable] - e_cmd_set_min_pub, - - // publish component price, never returning an error even if the update failed - // key[0] funding account [signer writable] - // key[1] price account [writable] - // key[2] sysvar_clock account [readable] - e_cmd_upd_price_no_fail_on_error, - - // resizes a price account so that it fits the Time Machine - // key[0] funding account [signer writable] - // key[1] price account [Signer writable] - // key[2] system program [readable] - e_cmd_resize_price_account, - - // deletes a price account - // key[0] funding account [signer writable] - // key[1] product account [signer writable] - // key[2] price account [signer writable] - e_cmd_del_price, - - // deletes a product account - // key[0] funding account [signer writable] - // key[1] mapping account [signer writable] - // key[2] product account [signer writable] - e_cmd_del_product, -} command_t; - -typedef struct cmd_hdr -{ - uint32_t ver_; - int32_t cmd_; -} cmd_hdr_t; - -static_assert( sizeof( cmd_hdr_t ) == 8, "" ); - -typedef struct cmd_add_product -{ - uint32_t ver_; - int32_t cmd_; -} cmd_add_product_t; - -static_assert( sizeof( cmd_add_product_t ) == 8, "" ); - -typedef struct cmd_upd_product -{ - uint32_t ver_; - int32_t cmd_; - // set of key-value pairs -} cmd_upd_product_t; - -static_assert( sizeof( cmd_upd_product_t ) == 8, "" ); - -typedef struct cmd_add_price -{ - uint32_t ver_; - int32_t cmd_; - int32_t expo_; - uint32_t ptype_; -} cmd_add_price_t; - -static_assert( sizeof( cmd_add_price_t ) == 16, "" ); - -typedef struct cmd_init_price -{ - uint32_t ver_; - int32_t cmd_; - int32_t expo_; - uint32_t ptype_; -} cmd_init_price_t; - -static_assert( sizeof( cmd_init_price_t ) == 16, "" ); - -typedef struct cmd_set_min_pub -{ - uint32_t ver_; - int32_t cmd_; - uint8_t min_pub_; -} cmd_set_min_pub_t; - -static_assert( sizeof( cmd_set_min_pub_t ) == 12, "" ); - -typedef struct cmd_add_publisher -{ - uint32_t ver_; - int32_t cmd_; - pc_pub_key_t pub_; -} cmd_add_publisher_t; - -static_assert( sizeof( cmd_add_publisher_t ) == 40, "" ); - -typedef struct cmd_del_publisher -{ - uint32_t ver_; - int32_t cmd_; - pc_pub_key_t pub_; -} cmd_del_publisher_t; - -static_assert( sizeof( cmd_del_publisher_t ) == 40, "" ); - -typedef struct cmd_upd_price -{ - uint32_t ver_; - int32_t cmd_; - uint32_t status_; - uint32_t unused_; - int64_t price_; - uint64_t conf_; - uint64_t pub_slot_; -} cmd_upd_price_t; - -static_assert( sizeof( cmd_upd_price_t ) == 40, "" ); - -// structure of clock sysvar account -typedef struct sysvar_clock -{ - uint64_t slot_; - int64_t epoch_start_timestamp_; - uint64_t epoch_; - uint64_t leader_schedule_epoch_; - int64_t unix_timestamp_; -} sysvar_clock_t; - -static_assert( sizeof( sysvar_clock_t ) == 40, "" ); - -// compare if two pub_keys (accounts) are the same -inline bool pc_pub_key_equal( pc_pub_key_t *p1, pc_pub_key_t *p2 ) -{ - return p1->k8_[0] == p2->k8_[0] && - p1->k8_[1] == p2->k8_[1] && - p1->k8_[2] == p2->k8_[2] && - p1->k8_[3] == p2->k8_[3]; -} - -// check for null (zero) public key -inline bool pc_pub_key_is_zero( pc_pub_key_t *p ) -{ - return p->k8_[0] == 0UL && - p->k8_[1] == 0UL && - p->k8_[2] == 0UL && - p->k8_[3] == 0UL; -} - -// set public key to zero -inline void pc_pub_key_set_zero( pc_pub_key_t *p ) -{ - p->k8_[0] = p->k8_[1] = p->k8_[2] = p->k8_[3] = 0UL; -} - -// assign one pub_key from another -inline void pc_pub_key_assign( pc_pub_key_t *tgt, pc_pub_key_t *src ) -{ - tgt->k8_[0] = src->k8_[0]; - tgt->k8_[1] = src->k8_[1]; - tgt->k8_[2] = src->k8_[2]; - tgt->k8_[3] = src->k8_[3]; -} - +#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (240 + PC_NUM_COMP * sizeof(pc_price_comp_t)) + + static_assert(sizeof(pc_price_t) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, ""); +#undef PC_EXPECTED_PRICE_T_SIZE_PYTHNET + + // This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd. + // It is set tighly to the current price account + 96 component prices + 48 bytes for cumulative sums + const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof(pc_price_comp_t) + 48; + + // command enumeration + typedef enum + { + + // initialize first mapping list account + // key[0] funding account [signer writable] + // key[1] mapping account [signer writable] + e_cmd_init_mapping = 0, + + // initialize and add new mapping account + // key[0] funding account [signer writable] + // key[1] tail mapping account [signer writable] + // key[2] new mapping account [signer writable] + e_cmd_add_mapping, + + // initialize and add new product reference data account + // key[0] funding account [signer writable] + // key[1] mapping account [signer writable] + // key[2] new product account [signer writable] + e_cmd_add_product, + + // update product account + // key[0] funding account [signer writable] + // key[1] product account [signer writable] + e_cmd_upd_product, + + // add new price account to a product account + // key[0] funding account [signer writable] + // key[1] product account [signer writable] + // key[2] new price account [signer writable] + e_cmd_add_price, + + // add publisher to symbol account + // key[0] funding account [signer writable] + // key[1] price account [signer writable] + e_cmd_add_publisher, + + // delete publisher from symbol account + // key[0] funding account [signer writable] + // key[1] price account [signer writable] + e_cmd_del_publisher, + + // publish component price + // key[0] funding account [signer writable] + // key[1] price account [writable] + // key[2] sysvar_clock account [readable] + e_cmd_upd_price, + + // compute aggregate price + // key[0] funding account [signer writable] + // key[1] price account [writable] + // key[2] sysvar_clock account [readable] + e_cmd_agg_price, + + // (re)initialize price account + // key[0] funding account [signer writable] + // key[1] new price account [signer writable] + e_cmd_init_price, + + // deprecated + e_cmd_init_test, + + // deprecated + e_cmd_upd_test, + + // set min publishers + // key[0] funding account [signer writable] + // key[1] price account [signer writable] + e_cmd_set_min_pub, + + // publish component price, never returning an error even if the update failed + // key[0] funding account [signer writable] + // key[1] price account [writable] + // key[2] sysvar_clock account [readable] + e_cmd_upd_price_no_fail_on_error, + + // resizes a price account so that it fits the Time Machine + // key[0] funding account [signer writable] + // key[1] price account [Signer writable] + // key[2] system program [readable] + e_cmd_resize_price_account, + + // deletes a price account + // key[0] funding account [signer writable] + // key[1] product account [signer writable] + // key[2] price account [signer writable] + e_cmd_del_price, + + // deletes a product account + // key[0] funding account [signer writable] + // key[1] mapping account [signer writable] + // key[2] product account [signer writable] + e_cmd_del_product, + } command_t; + + typedef struct cmd_hdr + { + uint32_t ver_; + int32_t cmd_; + } cmd_hdr_t; + + static_assert(sizeof(cmd_hdr_t) == 8, ""); + + typedef struct cmd_add_product + { + uint32_t ver_; + int32_t cmd_; + } cmd_add_product_t; + + static_assert(sizeof(cmd_add_product_t) == 8, ""); + + typedef struct cmd_upd_product + { + uint32_t ver_; + int32_t cmd_; + // set of key-value pairs + } cmd_upd_product_t; + + static_assert(sizeof(cmd_upd_product_t) == 8, ""); + + typedef struct cmd_add_price + { + uint32_t ver_; + int32_t cmd_; + int32_t expo_; + uint32_t ptype_; + } cmd_add_price_t; + + static_assert(sizeof(cmd_add_price_t) == 16, ""); + + typedef struct cmd_init_price + { + uint32_t ver_; + int32_t cmd_; + int32_t expo_; + uint32_t ptype_; + } cmd_init_price_t; + + static_assert(sizeof(cmd_init_price_t) == 16, ""); + + typedef struct cmd_set_min_pub + { + uint32_t ver_; + int32_t cmd_; + uint8_t min_pub_; + } cmd_set_min_pub_t; + + static_assert(sizeof(cmd_set_min_pub_t) == 12, ""); + + typedef struct cmd_add_publisher + { + uint32_t ver_; + int32_t cmd_; + pc_pub_key_t pub_; + } cmd_add_publisher_t; + + static_assert(sizeof(cmd_add_publisher_t) == 40, ""); + + typedef struct cmd_del_publisher + { + uint32_t ver_; + int32_t cmd_; + pc_pub_key_t pub_; + } cmd_del_publisher_t; + + static_assert(sizeof(cmd_del_publisher_t) == 40, ""); + + typedef struct cmd_upd_price + { + uint32_t ver_; + int32_t cmd_; + uint32_t status_; + uint32_t unused_; + int64_t price_; + uint64_t conf_; + uint64_t pub_slot_; + } cmd_upd_price_t; + + static_assert(sizeof(cmd_upd_price_t) == 40, ""); + + // structure of clock sysvar account + typedef struct sysvar_clock + { + uint64_t slot_; + int64_t epoch_start_timestamp_; + uint64_t epoch_; + uint64_t leader_schedule_epoch_; + int64_t unix_timestamp_; + } sysvar_clock_t; + + static_assert(sizeof(sysvar_clock_t) == 40, ""); + + // compare if two pub_keys (accounts) are the same + inline bool pc_pub_key_equal(pc_pub_key_t *p1, pc_pub_key_t *p2) + { + return p1->k8_[0] == p2->k8_[0] && + p1->k8_[1] == p2->k8_[1] && + p1->k8_[2] == p2->k8_[2] && + p1->k8_[3] == p2->k8_[3]; + } + + // check for null (zero) public key + inline bool pc_pub_key_is_zero(pc_pub_key_t *p) + { + return p->k8_[0] == 0UL && + p->k8_[1] == 0UL && + p->k8_[2] == 0UL && + p->k8_[3] == 0UL; + } + + // set public key to zero + inline void pc_pub_key_set_zero(pc_pub_key_t *p) + { + p->k8_[0] = p->k8_[1] = p->k8_[2] = p->k8_[3] = 0UL; + } + + // assign one pub_key from another + inline void pc_pub_key_assign(pc_pub_key_t *tgt, pc_pub_key_t *src) + { + tgt->k8_[0] = src->k8_[0]; + tgt->k8_[1] = src->k8_[1]; + tgt->k8_[2] = src->k8_[2]; + tgt->k8_[3] = src->k8_[3]; + } #ifdef __cplusplus } diff --git a/program/c/src/oracle/upd_aggregate.c b/program/c/src/oracle/upd_aggregate.c index 5b2a6af6..0c723a69 100644 --- a/program/c/src/oracle/upd_aggregate.c +++ b/program/c/src/oracle/upd_aggregate.c @@ -6,12 +6,8 @@ #include "upd_aggregate.h" #include "features.h" -// Dynamically deciding the name prevents linking the wrong C binary flavor -#ifdef PC_PYTHNET + extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#else -extern bool c_upd_aggregate_solana( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#endif return upd_aggregate(ptr, slot, timestamp ); } diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index ab77a070..1774845c 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -6,210 +6,221 @@ #include "pd.h" #ifdef __cplusplus -extern "C" { -#endif - -typedef struct pc_qset +extern "C" { - pd_t iprice_[PC_NUM_COMP]; - pd_t uprice_[PC_NUM_COMP]; - pd_t lprice_[PC_NUM_COMP]; - pd_t weight_[PC_NUM_COMP]; - int64_t decay_[1+PC_MAX_SEND_LATENCY]; - int64_t fact_[PC_FACTOR_SIZE]; -} pc_qset_t; - -// initialize quote-set temporary data in heap area -static pc_qset_t *qset_new() -{ - // allocate off heap - pc_qset_t *qs = (pc_qset_t*)PC_HEAP_START; - - // sqrt of numbers 1 to 25 for decaying conf. interval based on slot delay - qs->decay_[0] = 1000000000L; - qs->decay_[1] = 1000000000L; - qs->decay_[2] = 1414213562L; - qs->decay_[3] = 1732050807L; - qs->decay_[4] = 2000000000L; - qs->decay_[5] = 2236067977L; - qs->decay_[6] = 2449489742L; - qs->decay_[7] = 2645751311L; - qs->decay_[8] = 2828427124L; - qs->decay_[9] = 3000000000L; - qs->decay_[10] = 3162277660L; - qs->decay_[11] = 3316624790L; - qs->decay_[12] = 3464101615L; - qs->decay_[13] = 3605551275L; - qs->decay_[14] = 3741657386L; - qs->decay_[15] = 3872983346L; - qs->decay_[16] = 4000000000L; - qs->decay_[17] = 4123105625L; - qs->decay_[18] = 4242640687L; - qs->decay_[19] = 4358898943L; - qs->decay_[20] = 4472135954L; - qs->decay_[21] = 4582575694L; - qs->decay_[22] = 4690415759L; - qs->decay_[23] = 4795831523L; - qs->decay_[24] = 4898979485L; - qs->decay_[25] = 5000000000L; - - // powers of 10 for use in decimal arithmetic scaling - qs->fact_[0] = 1L; - qs->fact_[1] = 10L; - qs->fact_[2] = 100L; - qs->fact_[3] = 1000L; - qs->fact_[4] = 10000L; - qs->fact_[5] = 100000L; - qs->fact_[6] = 1000000L; - qs->fact_[7] = 10000000L; - qs->fact_[8] = 100000000L; - qs->fact_[9] = 1000000000L; - qs->fact_[10] = 10000000000L; - qs->fact_[11] = 100000000000L; - qs->fact_[12] = 1000000000000L; - qs->fact_[13] = 10000000000000L; - qs->fact_[14] = 100000000000000L; - qs->fact_[15] = 1000000000000000L; - qs->fact_[16] = 10000000000000000L; - qs->fact_[17] = 100000000000000000L; - - return qs; -} +#endif -static void upd_ema( - pc_ema_t *ptr, pd_t *val, pd_t *conf, int64_t nslot, pc_qset_t *qs, int32_t expo - ) -{ - pd_t numer[1], denom[1], cwgt[1], wval[1], decay[1], diff[1], one[1]; - pd_new( one, 100000000L, -8 ); - if ( conf->v_ ) { - pd_div( cwgt, one, conf ); - } else { - pd_set( cwgt, one ); - } - if ( nslot > PD_EMA_MAX_DIFF ) { - // initial condition - pd_mul( numer, val, cwgt ); - pd_set( denom, cwgt ); - } else { - // compute decay factor - pd_new( diff, nslot, 0 ); - pd_new( decay, PD_EMA_DECAY, PD_EMA_EXPO ); - pd_mul( decay, decay, diff ); - pd_add( decay, decay, one, qs->fact_ ); - - // compute numer/denom and new value from decay factor - pd_load( numer, ptr->numer_ ); - pd_load( denom, ptr->denom_ ); - pd_mul( numer, numer, decay ); - pd_mul( wval, val, cwgt ); - pd_add( numer, numer, wval, qs->fact_ ); - pd_mul( denom, denom, decay ); - pd_add( denom, denom, cwgt, qs->fact_ ); - pd_div( val, numer, denom ); + typedef struct pc_qset + { + pd_t iprice_[PC_NUM_COMP]; + pd_t uprice_[PC_NUM_COMP]; + pd_t lprice_[PC_NUM_COMP]; + pd_t weight_[PC_NUM_COMP]; + int64_t decay_[1 + PC_MAX_SEND_LATENCY]; + int64_t fact_[PC_FACTOR_SIZE]; + } pc_qset_t; + + // initialize quote-set temporary data in heap area + static pc_qset_t *qset_new() + { + // allocate off heap + pc_qset_t *qs = (pc_qset_t *)PC_HEAP_START; + + // sqrt of numbers 1 to 25 for decaying conf. interval based on slot delay + qs->decay_[0] = 1000000000L; + qs->decay_[1] = 1000000000L; + qs->decay_[2] = 1414213562L; + qs->decay_[3] = 1732050807L; + qs->decay_[4] = 2000000000L; + qs->decay_[5] = 2236067977L; + qs->decay_[6] = 2449489742L; + qs->decay_[7] = 2645751311L; + qs->decay_[8] = 2828427124L; + qs->decay_[9] = 3000000000L; + qs->decay_[10] = 3162277660L; + qs->decay_[11] = 3316624790L; + qs->decay_[12] = 3464101615L; + qs->decay_[13] = 3605551275L; + qs->decay_[14] = 3741657386L; + qs->decay_[15] = 3872983346L; + qs->decay_[16] = 4000000000L; + qs->decay_[17] = 4123105625L; + qs->decay_[18] = 4242640687L; + qs->decay_[19] = 4358898943L; + qs->decay_[20] = 4472135954L; + qs->decay_[21] = 4582575694L; + qs->decay_[22] = 4690415759L; + qs->decay_[23] = 4795831523L; + qs->decay_[24] = 4898979485L; + qs->decay_[25] = 5000000000L; + + // powers of 10 for use in decimal arithmetic scaling + qs->fact_[0] = 1L; + qs->fact_[1] = 10L; + qs->fact_[2] = 100L; + qs->fact_[3] = 1000L; + qs->fact_[4] = 10000L; + qs->fact_[5] = 100000L; + qs->fact_[6] = 1000000L; + qs->fact_[7] = 10000000L; + qs->fact_[8] = 100000000L; + qs->fact_[9] = 1000000000L; + qs->fact_[10] = 10000000000L; + qs->fact_[11] = 100000000000L; + qs->fact_[12] = 1000000000000L; + qs->fact_[13] = 10000000000000L; + qs->fact_[14] = 100000000000000L; + qs->fact_[15] = 1000000000000000L; + qs->fact_[16] = 10000000000000000L; + qs->fact_[17] = 100000000000000000L; + + return qs; } - // adjust and store results - pd_adjust( val, expo, qs->fact_ ); - ptr->val_ = val->v_; - int64_t numer1, denom1; - if ( pd_store( &numer1, numer ) && pd_store( &denom1, denom ) ) { - ptr->numer_ = numer1; - ptr->denom_ = denom1; + static void upd_ema( + pc_ema_t *ptr, pd_t *val, pd_t *conf, int64_t nslot, pc_qset_t *qs, int32_t expo) + { + pd_t numer[1], denom[1], cwgt[1], wval[1], decay[1], diff[1], one[1]; + pd_new(one, 100000000L, -8); + if (conf->v_) + { + pd_div(cwgt, one, conf); + } + else + { + pd_set(cwgt, one); + } + if (nslot > PD_EMA_MAX_DIFF) + { + // initial condition + pd_mul(numer, val, cwgt); + pd_set(denom, cwgt); + } + else + { + // compute decay factor + pd_new(diff, nslot, 0); + pd_new(decay, PD_EMA_DECAY, PD_EMA_EXPO); + pd_mul(decay, decay, diff); + pd_add(decay, decay, one, qs->fact_); + + // compute numer/denom and new value from decay factor + pd_load(numer, ptr->numer_); + pd_load(denom, ptr->denom_); + pd_mul(numer, numer, decay); + pd_mul(wval, val, cwgt); + pd_add(numer, numer, wval, qs->fact_); + pd_mul(denom, denom, decay); + pd_add(denom, denom, cwgt, qs->fact_); + pd_div(val, numer, denom); + } + + // adjust and store results + pd_adjust(val, expo, qs->fact_); + ptr->val_ = val->v_; + int64_t numer1, denom1; + if (pd_store(&numer1, numer) && pd_store(&denom1, denom)) + { + ptr->numer_ = numer1; + ptr->denom_ = denom1; + } } -} -static inline void upd_twap( - pc_price_t *ptr, int64_t nslots ) -{ - pc_qset_t *qs = qset_new( ); + static inline void upd_twap( + pc_price_t *ptr, int64_t nslots) + { + pc_qset_t *qs = qset_new(); - pd_t px[1], conf[1]; - pd_new_scale( px, ptr->agg_.price_, ptr->expo_ ); - pd_new_scale( conf, ( int64_t )( ptr->agg_.conf_ ), ptr->expo_ ); - upd_ema( &ptr->twap_, px, conf, nslots, qs, ptr->expo_ ); - upd_ema( &ptr->twac_, conf, conf, nslots, qs, ptr->expo_ ); -} + pd_t px[1], conf[1]; + pd_new_scale(px, ptr->agg_.price_, ptr->expo_); + pd_new_scale(conf, (int64_t)(ptr->agg_.conf_), ptr->expo_); + upd_ema(&ptr->twap_, px, conf, nslots, qs, ptr->expo_); + upd_ema(&ptr->twac_, conf, conf, nslots, qs, ptr->expo_); + } -// update aggregate price -static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timestamp ) -{ - // update aggregate details ready for next slot - ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price - ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price - ptr->timestamp_ = timestamp; - - // identify valid quotes - // compute the aggregate prices and ranges - int64_t agg_price; - int64_t agg_conf; + // update aggregate price + static inline bool upd_aggregate(pc_price_t *ptr, uint64_t slot, int64_t timestamp) { - uint32_t numv = 0; - uint32_t nprcs = (uint32_t)0; - int64_t prcs[ PC_NUM_COMP * 3 ]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) - for ( uint32_t i = 0; i != ptr->num_; ++i ) { - pc_price_comp_t *iptr = &ptr->comp_[i]; - // copy contributing price to aggregate snapshot - iptr->agg_ = iptr->latest_; - // add quote to sorted permutation array if it is valid - int64_t slot_diff = ( int64_t )slot - ( int64_t )( iptr->agg_.pub_slot_ ); - int64_t price = iptr->agg_.price_; - int64_t conf = ( int64_t )( iptr->agg_.conf_ ); - int64_t max_latency = ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY; - if ( iptr->agg_.status_ == PC_STATUS_TRADING && - // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < INT64_MAX - // These checks ensure that price - conf and price + conf do not overflow. - (int64_t)0 < conf && (INT64_MIN + conf) <= price && price <= (INT64_MAX-conf) && - // slot_diff is implicitly >= 0 due to the check in Rust code ensuring publishing_slot is always less than or equal to the current slot. - slot_diff <= max_latency ) { - numv += 1; - prcs[ nprcs++ ] = price - conf; - prcs[ nprcs++ ] = price; - prcs[ nprcs++ ] = price + conf; + // update aggregate details ready for next slot + ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price + ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price + ptr->timestamp_ = timestamp; + + // identify valid quotes + // compute the aggregate prices and ranges + int64_t agg_price; + int64_t agg_conf; + { + uint32_t numv = 0; + uint32_t nprcs = (uint32_t)0; + int64_t prcs[PC_NUM_COMP * 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + for (uint32_t i = 0; i != ptr->num_; ++i) + { + pc_price_comp_t *iptr = &ptr->comp_[i]; + // copy contributing price to aggregate snapshot + iptr->agg_ = iptr->latest_; + // add quote to sorted permutation array if it is valid + int64_t slot_diff = (int64_t)slot - (int64_t)(iptr->agg_.pub_slot_); + int64_t price = iptr->agg_.price_; + int64_t conf = (int64_t)(iptr->agg_.conf_); + int64_t max_latency = ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY; + if (iptr->agg_.status_ == PC_STATUS_TRADING && + // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < INT64_MAX + // These checks ensure that price - conf and price + conf do not overflow. + (int64_t)0 < conf && (INT64_MIN + conf) <= price && price <= (INT64_MAX - conf) && + // slot_diff is implicitly >= 0 due to the check in Rust code ensuring publishing_slot is always less than or equal to the current slot. + slot_diff <= max_latency) + { + numv += 1; + prcs[nprcs++] = price - conf; + prcs[nprcs++] = price; + prcs[nprcs++] = price + conf; + } } - } - // too few valid quotes - ptr->num_qt_ = numv; - if ( numv == 0 || numv < ptr->min_pub_ ) { - ptr->agg_.status_ = PC_STATUS_UNKNOWN; - return false; - } + // too few valid quotes + ptr->num_qt_ = numv; + if (numv == 0 || numv < ptr->min_pub_) + { + ptr->agg_.status_ = PC_STATUS_UNKNOWN; + return false; + } - // evaluate the model to get the p25/p50/p75 prices - // note: numv>0 and nprcs = 3*numv at this point - int64_t agg_p25; - int64_t agg_p75; - int64_t scratch[ PC_NUM_COMP * 3 ]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) - price_model_core( (uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, scratch ); - - // get the left and right confidences - // note that as valid quotes have positive prices currently and - // agg_p25, agg_price, agg_p75 are ordered, this calculation can't - // overflow / underflow - int64_t agg_conf_left = agg_price - agg_p25; - int64_t agg_conf_right = agg_p75 - agg_price; - - // use the larger of the left and right confidences - agg_conf = agg_conf_right > agg_conf_left ? agg_conf_right : agg_conf_left; - - // if the confidences end up at zero, we abort - // this is paranoia as it is currently not possible when nprcs>2 and - // positive confidences given the current pricing model - if( agg_conf <= (int64_t)0 ) { - ptr->agg_.status_ = PC_STATUS_UNKNOWN; - return false; + // evaluate the model to get the p25/p50/p75 prices + // note: numv>0 and nprcs = 3*numv at this point + int64_t agg_p25; + int64_t agg_p75; + int64_t scratch[PC_NUM_COMP * 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + price_model_core((uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, scratch); + + // get the left and right confidences + // note that as valid quotes have positive prices currently and + // agg_p25, agg_price, agg_p75 are ordered, this calculation can't + // overflow / underflow + int64_t agg_conf_left = agg_price - agg_p25; + int64_t agg_conf_right = agg_p75 - agg_price; + + // use the larger of the left and right confidences + agg_conf = agg_conf_right > agg_conf_left ? agg_conf_right : agg_conf_left; + + // if the confidences end up at zero, we abort + // this is paranoia as it is currently not possible when nprcs>2 and + // positive confidences given the current pricing model + if (agg_conf <= (int64_t)0) + { + ptr->agg_.status_ = PC_STATUS_UNKNOWN; + return false; + } } - } - // update status and publish slot of last trading status price - ptr->agg_.status_ = PC_STATUS_TRADING; - ptr->last_slot_ = slot; - ptr->agg_.price_ = agg_price; - ptr->agg_.conf_ = (uint64_t)agg_conf; + // update status and publish slot of last trading status price + ptr->agg_.status_ = PC_STATUS_TRADING; + ptr->last_slot_ = slot; + ptr->agg_.price_ = agg_price; + ptr->agg_.conf_ = (uint64_t)agg_conf; - return true; -} + return true; + } #ifdef __cplusplus } diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 1c2ff744..37c045ef 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -43,11 +43,9 @@ lazy_static = "1.4.0" # to cargo-build-bpf at that point - we manually capture them at # compile-time and pass on to the child process. [features] -default = ["pythnet"] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] library = [] -pythnet = [] # logic-affecting features start with this one [lib] crate-type = ["cdylib", "lib"] diff --git a/program/rust/build.rs b/program/rust/build.rs index 7d746c0a..b33d8dd7 100644 --- a/program/rust/build.rs +++ b/program/rust/build.rs @@ -12,7 +12,6 @@ use { fn main() { let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - let has_feat_pythnet = std::env::var("CARGO_FEATURE_PYTHNET").is_ok(); let has_feat_check = std::env::var("CARGO_FEATURE_CHECK").is_ok(); // OUT_DIR is the path cargo provides to a build directory under `target/` specifically for @@ -25,17 +24,6 @@ fn main() { eprintln!("OUT_DIR is {}", out_dir); let out_dir = PathBuf::from(out_dir); - let mut make_extra_flags = vec![]; - let mut clang_extra_flags = vec![]; - - if has_feat_pythnet { - // Define PC_PYTHNET for the C binary build - make_extra_flags.push("PC_PYTHNET=1"); - - // Define PC_PYTHNET for the bindings build - clang_extra_flags.push("-DPC_PYTHNET=1"); - } - let mut make_targets = vec![]; if target_arch == "bpf" { make_targets.push("cpyth-bpf"); @@ -51,7 +39,7 @@ fn main() { if has_feat_check { eprintln!("WARNING: `check` feature active, make build is skipped"); } else { - do_make_build(make_extra_flags, make_targets, &out_dir); + do_make_build(make_targets, &out_dir); // Link against the right library for the architecture if target_arch == "bpf" { @@ -74,7 +62,6 @@ fn main() { // Generate and write bindings let bindings = Builder::default() .clang_arg(format!("-I{:}", get_solana_inc_path().display())) - .clang_args(clang_extra_flags) .header("./src/bindings.h") .rustfmt_bindings(true) .generate() @@ -88,14 +75,13 @@ fn main() { println!("cargo:rerun-if-changed=../"); } -fn do_make_build(extra_flags: Vec<&str>, targets: Vec<&str>, out_dir: &Path) { +fn do_make_build(targets: Vec<&str>, out_dir: &Path) { // We must forward OUT_DIR as an env variable to the make script otherwise it will output // its artifacts to the wrong place. let make_output = std::process::Command::new("make") .env("VERBOSE", "1") .env("OUT_DIR", out_dir.display().to_string()) .current_dir("../c") - .args(extra_flags) .args(targets) .output() .expect("Failed to run make for C oracle program"); diff --git a/program/rust/src/accounts.rs b/program/rust/src/accounts.rs index 07cb5a05..5958fc88 100644 --- a/program/rust/src/accounts.rs +++ b/program/rust/src/accounts.rs @@ -43,8 +43,6 @@ mod product; // Some types only exist during use as a library. #[cfg(feature = "strum")] pub use price::MessageType; -#[cfg(feature = "pythnet")] -pub use price::PriceCumulative; #[cfg(test)] pub use product::{ account_has_key_values, @@ -56,6 +54,7 @@ pub use { price::{ PriceAccount, PriceComponent, + PriceCumulative, PriceEma, PriceInfo, PythOracleSerialize, @@ -72,7 +71,6 @@ pub use { /// are authorized to perform certain administrative actions. pub const PERMISSIONS_SEED: &str = "permissions"; -#[cfg(feature = "pythnet")] /// The update price instruction can optionally invoke another program via CPI. The /// CPI will be signed with the PDA `[UPD_PRICE_WRITE_SEED, invoked_program_public_key]` /// such that the caller can authenticate its origin. diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index f0fa23a7..bcc76a59 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -1,7 +1,4 @@ -#[cfg(feature = "pythnet")] pub use price_pythnet::*; -#[cfg(not(feature = "pythnet"))] -pub use price_solana::*; use { super::{ AccountHeader, @@ -21,7 +18,6 @@ use { }; /// Pythnet-specific PriceAccount implementation -#[cfg(feature = "pythnet")] mod price_pythnet { pub type PriceAccount = PriceAccountPythnet; @@ -242,79 +238,6 @@ mod price_pythnet { } } -/// Solana-specific PriceAccount implementation -#[cfg(not(feature = "pythnet"))] -mod price_solana { - pub type PriceAccount = PriceAccountSolana; - - use { - super::*, - crate::c_oracle_header::PC_NUM_COMP_SOLANA, - }; - - /// Legacy Solana-only price account format. This price account - /// schema is maintained for compatibility reasons. - /// - /// The main source of incompatibility on mainnet/devnet is the - /// possibility of strict account size checks which could break - /// existing users if they encountered a longer struct like - /// PriceAccountPythnet (see mod price_pythnet for details) - #[repr(C)] - #[derive(Copy, Clone, Pod, Zeroable)] - pub struct PriceAccountSolana { - pub header: AccountHeader, - /// Type of the price account - pub price_type: u32, - /// Exponent for the published prices - pub exponent: i32, - /// Current number of authorized publishers - pub num_: u32, - /// Number of valid quotes for the last aggregation - pub num_qt_: u32, - /// Last slot with a succesful aggregation (status : TRADING) - pub last_slot_: u64, - /// Second to last slot where aggregation was attempted - pub valid_slot_: u64, - /// Ema for price - pub twap_: PriceEma, - /// Ema for confidence - pub twac_: PriceEma, - /// Last time aggregation was attempted - pub timestamp_: i64, - /// Minimum valid publisher quotes for a succesful aggregation - pub min_pub_: u8, - /// Whether the current aggregate price has been sent as a message to the message buffer. - /// 0 = false, 1 = true. (this is a u8 to make the Pod trait happy) - pub message_sent_: u8, - /// Configurable max latency in slots between send and receive - pub max_latency_: u8, - /// Unused placeholder for alignment - pub unused_2_: i8, - pub unused_3_: i32, - /// Corresponding product account - pub product_account: Pubkey, - /// Next price account in the list - pub next_price_account: Pubkey, - /// Second to last slot where aggregation was succesful (i.e. status : TRADING) - pub prev_slot_: u64, - /// Aggregate price at prev_slot_ - pub prev_price_: i64, - /// Confidence interval at prev_slot_ - pub prev_conf_: u64, - /// Timestamp of prev_slot_ - pub prev_timestamp_: i64, - /// Last attempted aggregate results - pub agg_: PriceInfo, - /// Publishers' price components - pub comp_: [PriceComponent; PC_NUM_COMP_SOLANA as usize], - } - - impl PythAccount for PriceAccountSolana { - const ACCOUNT_TYPE: u32 = PC_ACCTYPE_PRICE; - const INITIAL_SIZE: u32 = size_of::() as u32; - } -} - #[repr(C)] #[derive(Copy, Clone, Pod, Zeroable)] pub struct PriceComponent { diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 7a4b5599..0c5669f2 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -1,20 +1,10 @@ -#[cfg(feature = "pythnet")] -use { - crate::accounts::{ - PythOracleSerialize, - UPD_PRICE_WRITE_SEED, - }, - solana_program::instruction::{ - AccountMeta, - Instruction, - }, - solana_program::program::invoke_signed, -}; use { crate::{ accounts::{ PriceAccount, PriceInfo, + PythOracleSerialize, + UPD_PRICE_WRITE_SEED, }, c_oracle_header::PC_STATUS_TRADING, deserialize::{ @@ -36,6 +26,11 @@ use { account_info::AccountInfo, clock::Clock, entrypoint::ProgramResult, + instruction::{ + AccountMeta, + Instruction, + }, + program::invoke_signed, program_error::ProgramError, pubkey::Pubkey, sysvar::Sysvar, @@ -45,10 +40,7 @@ use { #[cfg(target_arch = "bpf")] #[link(name = "cpyth-bpf")] extern "C" { - #[cfg(feature = "pythnet")] pub fn c_upd_aggregate_pythnet(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; - #[cfg(not(feature = "pythnet"))] - pub fn c_upd_aggregate_solana(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; #[allow(unused)] pub fn c_upd_twap(_input: *mut u8, nslots: i64); @@ -57,10 +49,7 @@ extern "C" { #[cfg(not(target_arch = "bpf"))] #[link(name = "cpyth-native")] extern "C" { - #[cfg(feature = "pythnet")] pub fn c_upd_aggregate_pythnet(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; - #[cfg(not(feature = "pythnet"))] - pub fn c_upd_aggregate_solana(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; #[allow(unused)] pub fn c_upd_twap(_input: *mut u8, nslots: i64); @@ -68,10 +57,7 @@ extern "C" { #[inline] pub unsafe fn c_upd_aggregate(input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool { - #[cfg(feature = "pythnet")] - return c_upd_aggregate_pythnet(input, clock_slot, clock_timestamp); - #[cfg(not(feature = "pythnet"))] - return c_upd_aggregate_solana(input, clock_slot, clock_timestamp); + c_upd_aggregate_pythnet(input, clock_slot, clock_timestamp) } /// Publish component price, never returning an error even if the update failed @@ -89,9 +75,9 @@ pub fn upd_price_no_fail_on_error( } } -/// Update a publisher's price for the provided product. This operation's behavior varies based on the feature flag: -/// - For `feature = "pythnet"`, price aggregation will trigger on the same slot for every price update, resulting in a new aggregate price in the account. -/// - For non-`pythnet` configurations, the first price update in a slot triggers price aggregation for the prices of the previous slot, resulting in a new aggregate price in the account. +/// Update a publisher's price for the provided product. If this update is +/// the first update in a slot, this operation will also trigger price aggregation +/// and result in a new aggregate price in the account. /// /// account[0] the publisher's account (funds the tx) [signer writable] /// fails if the publisher's public key is not permissioned for the price account. @@ -176,68 +162,55 @@ pub fn upd_price( )?; } - #[cfg(feature = "pythnet")] - { - // Reload price data as a struct after c_upd_aggregate() borrow is dropped - let mut price_data = load_checked::(price_account, cmd_args.header.version)?; + // Try to update the publisher's price + if is_component_update(cmd_args)? { + // IMPORTANT: If the publisher does not meet the price/conf + // ratio condition, its price will not count for the next + // aggregate. + let status: u32 = + get_status_for_conf_price_ratio(cmd_args.price, cmd_args.confidence, cmd_args.status)?; - // Try to update the publisher's price - if is_component_update(cmd_args)? { - // IMPORTANT: If the publisher does not meet the price/conf - // ratio condition, its price will not count for the next - // aggregate. - let status: u32 = get_status_for_conf_price_ratio( - cmd_args.price, - cmd_args.confidence, - cmd_args.status, - )?; - - { - let publisher_price = &mut price_data.comp_[publisher_index].latest_; - publisher_price.price_ = cmd_args.price; - publisher_price.conf_ = cmd_args.confidence; - publisher_price.status_ = status; - publisher_price.pub_slot_ = cmd_args.publishing_slot; - } + { + let mut price_data = + load_checked::(price_account, cmd_args.header.version)?; + let publisher_price = &mut price_data.comp_[publisher_index].latest_; + publisher_price.price_ = cmd_args.price; + publisher_price.conf_ = cmd_args.confidence; + publisher_price.status_ = status; + publisher_price.pub_slot_ = cmd_args.publishing_slot; } } - let should_call_c_upd_aggregate: bool = if cfg!(feature = "pythnet") { - true // clock.slot >= latest_aggregate_price.pub_slot_ is always true - } else { - clock.slot > latest_aggregate_price.pub_slot_ - }; - // get number of slots from last update let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; - let aggregate_updated = if should_call_c_upd_aggregate { - if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { - let mut price_data = - load_checked::(price_account, cmd_args.header.version)?; - price_data.prev_slot_ = price_data.agg_.pub_slot_; - price_data.prev_price_ = price_data.agg_.price_; - price_data.prev_conf_ = price_data.agg_.conf_; - price_data.prev_timestamp_ = clock.unix_timestamp; - } - unsafe { - c_upd_aggregate( - price_account.try_borrow_mut_data()?.as_mut_ptr(), - clock.slot, - clock.unix_timestamp, - ) - } - } else { - false // Do not call c_upd_aggregate and set aggregate_updated to false + // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. + if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { + let mut price_data = load_checked::(price_account, cmd_args.header.version)?; + price_data.prev_slot_ = price_data.agg_.pub_slot_; + price_data.prev_price_ = price_data.agg_.price_; + price_data.prev_conf_ = price_data.agg_.conf_; + price_data.prev_timestamp_ = clock.unix_timestamp; }; - if aggregate_updated { - // The price_data borrow happens in a scope because it must be - // dropped before we borrow again as raw data pointer for the C - // aggregation logic. - #[cfg(feature = "pythnet")] + + let updated = unsafe { + // NOTE: c_upd_aggregate must use a raw pointer to price + // data. Solana's `.borrow_*` methods require exclusive + // access, i.e. no other borrow can exist for the account. + c_upd_aggregate( + price_account.try_borrow_mut_data()?.as_mut_ptr(), + clock.slot, + clock.unix_timestamp, + ) + }; + + // If the aggregate was successfully updated, calculate the difference and update TWAP. + if updated { + let agg_diff = (clock.slot as i64) + - load_checked::(price_account, cmd_args.header.version)?.prev_slot_ + as i64; { - // get price data here let mut price_data = load_checked::(price_account, cmd_args.header.version)?; // check if its the first price update in the slot @@ -250,35 +223,24 @@ pub fn upd_price( price_data.twac_ = price_data.prev_twac_; price_data.price_cumulative = price_data.prev_price_cumulative; price_data.update_price_cumulative()?; + // We want to send a message every time the aggregate price updates. However, during the migration, + // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag + // ensures that after every aggregate update, the next publisher who provides the accumulator accounts + // will send the message. + price_data.message_sent_ = 0; } + // Encapsulate TWAP update logic in a function to minimize unsafe block scope. unsafe { - let prev_slot = { - // get price data here - let new_price_data = - load_checked::(price_account, cmd_args.header.version)?; - new_price_data.prev_slot_ - }; - c_upd_twap( - price_account.try_borrow_mut_data()?.as_mut_ptr(), - (clock.slot - prev_slot) as i64, // Ensure slots_since_last_update is cast to i64, as expected by the function signature - ); + c_upd_twap(price_account.try_borrow_mut_data()?.as_mut_ptr(), agg_diff); } } + // Reload price data as a struct after c_upd_aggregate() borrow is dropped let mut price_data = load_checked::(price_account, cmd_args.header.version)?; // Feature-gated accumulator-specific code, used only on pythnet/pythtest - #[cfg(feature = "pythnet")] { - // We want to send a message every time the aggregate price updates. However, during the migration, - // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag - // ensures that after every aggregate update, the next publisher who provides the accumulator accounts - // will send the message. - if aggregate_updated { - price_data.message_sent_ = 0; - } - if let Some(accumulator_accounts) = maybe_accumulator_accounts { if price_data.message_sent_ == 0 { // Check that the oracle PDA is correctly configured for the program we are calling. @@ -340,24 +302,6 @@ pub fn upd_price( } } - #[cfg(not(feature = "pythnet"))] - // Try to update the publisher's price - if is_component_update(cmd_args)? { - // IMPORTANT: If the publisher does not meet the price/conf - // ratio condition, its price will not count for the next - // aggregate. - let status: u32 = - get_status_for_conf_price_ratio(cmd_args.price, cmd_args.confidence, cmd_args.status)?; - - { - let publisher_price = &mut price_data.comp_[publisher_index].latest_; - publisher_price.price_ = cmd_args.price; - publisher_price.conf_ = cmd_args.confidence; - publisher_price.status_ = status; - publisher_price.pub_slot_ = cmd_args.publishing_slot; - } - } - Ok(()) } diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index 1263ae60..59ae7e5b 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -26,7 +26,5 @@ mod test_upd_price_no_fail_on_error; mod test_upd_product; mod test_utils; -#[cfg(feature = "pythnet")] mod test_twap; -#[cfg(feature = "pythnet")] mod test_upd_price_v2; diff --git a/program/rust/src/tests/pyth_simulator.rs b/program/rust/src/tests/pyth_simulator.rs index 6a77ba0c..87640503 100644 --- a/program/rust/src/tests/pyth_simulator.rs +++ b/program/rust/src/tests/pyth_simulator.rs @@ -70,43 +70,6 @@ use { }, }; -lazy_static::lazy_static! { - // Build the oracle binary and make it available to the - // simulator. lazy_static makes this happen only once per test - // run. - static ref ORACLE_PROGRAM_BINARY_PATH: PathBuf = { - - // Detect features and pass them onto cargo-build-bpf. - // IMPORTANT: All features of this crate must have gates added to this vector. - let features: Vec<&str> = vec![ - #[cfg(feature = "pythnet")] - "pythnet", - - ]; - - let mut cmd = std::process::Command::new("cargo"); - cmd.arg("build-bpf"); - - if !features.is_empty() { - cmd.arg("--features"); - cmd.args(features); - } - - let status = cmd.status().unwrap(); - - if !status.success() { - panic!( - "cargo-build-bpf did not exit with 0 (code {:?})", - status.code() - ); - } - - let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); - - PathBuf::from(target_dir).join("deploy/pyth_oracle.so") - }; -} - /// Simulator for the state of the pyth program on Solana. You can run solana transactions against /// this struct to test how pyth instructions execute in the Solana runtime. pub struct PythSimulator { @@ -139,7 +102,9 @@ struct ProductMetadata { impl PythSimulator { /// Deploys the oracle program as upgradable pub async fn new() -> PythSimulator { - let mut bpf_data = read_file(&*ORACLE_PROGRAM_BINARY_PATH); + let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); + let oracle_program_binary_path = PathBuf::from(target_dir).join("deploy/pyth_oracle.so"); + let mut bpf_data = read_file(&oracle_program_binary_path); let mut program_test = ProgramTest::default(); let program_key = Pubkey::new_unique(); @@ -215,7 +180,6 @@ impl PythSimulator { .await .unwrap(); - result } diff --git a/program/rust/src/tests/test_publish.rs b/program/rust/src/tests/test_publish.rs index 3ffc52ad..26eb463b 100644 --- a/program/rust/src/tests/test_publish.rs +++ b/program/rust/src/tests/test_publish.rs @@ -64,32 +64,6 @@ async fn test_publish() { .await .unwrap(); - #[cfg(not(feature = "pythnet"))] - { - let price_data = sim - .get_account_data_as::(price) - .await - .unwrap(); - - assert_eq!(price_data.comp_[0].latest_.price_, 150); - assert_eq!(price_data.comp_[0].latest_.conf_, 7); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - - #[cfg(not(feature = "pythnet"))] - { - assert_eq!(price_data.comp_[0].agg_.price_, 0); - assert_eq!(price_data.comp_[0].agg_.conf_, 0); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); - } - - #[cfg(feature = "pythnet")] - { - assert_eq!(price_data.comp_[0].agg_.price_, 150); - assert_eq!(price_data.comp_[0].agg_.conf_, 7); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); - } - } - sim.warp_to_slot(2).await.unwrap(); sim.upd_price( &publisher, @@ -113,18 +87,8 @@ async fn test_publish() { assert_eq!(price_data.comp_[0].latest_.conf_, 0); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_UNKNOWN); - #[cfg(not(feature = "pythnet"))] - { - assert_eq!(price_data.comp_[0].agg_.price_, 150); - assert_eq!(price_data.comp_[0].agg_.conf_, 7); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); - } - - #[cfg(feature = "pythnet")] - { - assert_eq!(price_data.comp_[0].agg_.price_, 0); - assert_eq!(price_data.comp_[0].agg_.conf_, 0); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); - } + assert_eq!(price_data.comp_[0].agg_.price_, 0); + assert_eq!(price_data.comp_[0].agg_.conf_, 0); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); } } diff --git a/program/rust/src/tests/test_publish_batch.rs b/program/rust/src/tests/test_publish_batch.rs index 5f6b8f05..f40f97b5 100644 --- a/program/rust/src/tests/test_publish_batch.rs +++ b/program/rust/src/tests/test_publish_batch.rs @@ -84,22 +84,12 @@ async fn test_publish_batch() { price_data.comp_[0].latest_.status_, get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status).unwrap() ); - #[cfg(not(feature = "pythnet"))] - { - assert_eq!(price_data.comp_[0].agg_.price_, 0); - assert_eq!(price_data.comp_[0].agg_.conf_, 0); - assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_UNKNOWN); - } - #[cfg(feature = "pythnet")] - { - assert_eq!(price_data.comp_[0].agg_.price_, quote.price); - assert_eq!(price_data.comp_[0].agg_.conf_, quote.confidence); - assert_eq!( - price_data.comp_[0].agg_.status_, - get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status) - .unwrap() - ); - } + assert_eq!(price_data.comp_[0].agg_.price_, quote.price); + assert_eq!(price_data.comp_[0].agg_.conf_, quote.confidence); + assert_eq!( + price_data.comp_[0].agg_.status_, + get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status).unwrap() + ); } sim.warp_to_slot(2).await.unwrap(); @@ -137,30 +127,16 @@ async fn test_publish_batch() { ) .unwrap() ); - #[cfg(not(feature = "pythnet"))] - { - let quote = quotes.get(key).unwrap(); - assert_eq!(price_data.comp_[0].agg_.price_, quote.price); - assert_eq!(price_data.comp_[0].agg_.conf_, quote.confidence); - assert_eq!( - price_data.comp_[0].agg_.status_, - get_status_for_conf_price_ratio(quote.price, quote.confidence, quote.status) - .unwrap() - ); - } - #[cfg(feature = "pythnet")] - { - assert_eq!(price_data.comp_[0].agg_.price_, new_quote.price); - assert_eq!(price_data.comp_[0].agg_.conf_, new_quote.confidence); - assert_eq!( - price_data.comp_[0].agg_.status_, - get_status_for_conf_price_ratio( - new_quote.price, - new_quote.confidence, - new_quote.status - ) - .unwrap() - ); - } + assert_eq!(price_data.comp_[0].agg_.price_, new_quote.price); + assert_eq!(price_data.comp_[0].agg_.conf_, new_quote.confidence); + assert_eq!( + price_data.comp_[0].agg_.status_, + get_status_for_conf_price_ratio( + new_quote.price, + new_quote.confidence, + new_quote.status + ) + .unwrap() + ); } } diff --git a/program/rust/src/tests/test_sizes.rs b/program/rust/src/tests/test_sizes.rs index b3ec7507..50e68341 100644 --- a/program/rust/src/tests/test_sizes.rs +++ b/program/rust/src/tests/test_sizes.rs @@ -6,6 +6,7 @@ use { PermissionAccount, PriceAccount, PriceComponent, + PriceCumulative, PriceEma, PriceInfo, ProductAccount, @@ -14,6 +15,7 @@ use { c_oracle_header::{ PC_MAP_TABLE_SIZE, PC_NUM_COMP, + PC_NUM_COMP_PYTHNET, PC_VERSION, ZSTD_UPPER_BOUND, }, @@ -52,52 +54,22 @@ fn test_sizes() { size_of::(), size_of::() + 2 * size_of::() ); - - #[cfg(feature = "pythnet")] - { - use crate::{ - accounts::PriceCumulative, - c_oracle_header::PC_NUM_COMP_PYTHNET, - }; - - // Sanity-check the Pythnet PC_NUM_COMP - assert_eq!(PC_NUM_COMP, 64); - - assert_eq!( - size_of::(), - 48 + u64::BITS as usize - + 3 * size_of::() - + size_of::() - + (PC_NUM_COMP_PYTHNET as usize) * size_of::() - + size_of::() - + size_of::() - + size_of::() - + size_of::() - ); - assert_eq!(size_of::(), 12576); - assert!(size_of::() == try_convert::<_, usize>(ZSTD_UPPER_BOUND).unwrap()); - - assert_eq!(size_of::(), 48); - } - - #[cfg(not(feature = "pythnet"))] - { - use crate::c_oracle_header::PC_NUM_COMP_SOLANA; - - // Sanity-check the Solana PC_NUM_COMP - assert_eq!(PC_NUM_COMP, PC_NUM_COMP_SOLANA); - - assert_eq!( - size_of::(), - 48 + u64::BITS as usize - + 3 * size_of::() - + size_of::() - + (PC_NUM_COMP_SOLANA as usize) * size_of::() - ); - assert_eq!(size_of::(), 3312); - assert!(size_of::() <= try_convert::<_, usize>(ZSTD_UPPER_BOUND).unwrap()); - } - + // Sanity-check the Pythnet PC_NUM_COMP + assert_eq!(PC_NUM_COMP, 64); + assert_eq!( + size_of::(), + 48 + u64::BITS as usize + + 3 * size_of::() + + size_of::() + + (PC_NUM_COMP_PYTHNET as usize) * size_of::() + + size_of::() + + size_of::() + + size_of::() + + size_of::() + ); + assert_eq!(size_of::(), 12576); + assert!(size_of::() == try_convert::<_, usize>(ZSTD_UPPER_BOUND).unwrap()); + assert_eq!(size_of::(), 48); assert_eq!(size_of::(), 8); assert_eq!(size_of::(), 16); assert_eq!(size_of::(), 16); diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index a92986b5..55290d85 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -70,20 +70,6 @@ fn test_upd_price() { ) .is_ok()); - #[cfg(not(feature = "pythnet"))] - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 42); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 0); - assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - } - - #[cfg(feature = "pythnet")] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 42); @@ -112,20 +98,6 @@ fn test_upd_price() { Err(ProgramError::InvalidArgument) ); - #[cfg(not(feature = "pythnet"))] - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 42); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 0); - assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - } - - #[cfg(feature = "pythnet")] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 42); @@ -153,20 +125,6 @@ fn test_upd_price() { ) .is_ok()); - #[cfg(not(feature = "pythnet"))] - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 81); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 2); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 1); - assert_eq!(price_data.agg_.pub_slot_, 3); - assert_eq!(price_data.agg_.price_, 42); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - } - - #[cfg(feature = "pythnet")] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 81); @@ -281,20 +239,6 @@ fn test_upd_price() { ) .is_ok()); - #[cfg(not(feature = "pythnet"))] - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 50); - assert_eq!(price_data.comp_[0].latest_.conf_, 20); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 5); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); - assert_eq!(price_data.valid_slot_, 5); - assert_eq!(price_data.agg_.pub_slot_, 6); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - } - - #[cfg(feature = "pythnet")] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, 50); @@ -348,21 +292,6 @@ fn test_upd_price() { &instruction_data ) .is_ok()); - - #[cfg(not(feature = "pythnet"))] - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, -100); - assert_eq!(price_data.comp_[0].latest_.conf_, 1); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 7); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 7); - assert_eq!(price_data.agg_.pub_slot_, 8); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - } - - #[cfg(feature = "pythnet")] { let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.comp_[0].latest_.price_, -100); diff --git a/program/rust/src/tests/test_upd_price_no_fail_on_error.rs b/program/rust/src/tests/test_upd_price_no_fail_on_error.rs index a3ebed02..1d175d8b 100644 --- a/program/rust/src/tests/test_upd_price_no_fail_on_error.rs +++ b/program/rust/src/tests/test_upd_price_no_fail_on_error.rs @@ -119,16 +119,8 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); assert_eq!(price_data.agg_.pub_slot_, 1); - #[cfg(not(feature = "pythnet"))] - { - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - } - #[cfg(feature = "pythnet")] - { - assert_eq!(price_data.agg_.price_, 42); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - } + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); } // Invalid updates, such as publishing an update for the current slot, @@ -169,16 +161,8 @@ fn test_upd_price_no_fail_on_error_no_fail_on_error() { assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); assert_eq!(price_data.agg_.pub_slot_, 1); - #[cfg(not(feature = "pythnet"))] - { - assert_eq!(price_data.agg_.price_, 0); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - } - #[cfg(feature = "pythnet")] - { - assert_eq!(price_data.agg_.price_, 42); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - } + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); } } diff --git a/scripts/build-bpf.sh b/scripts/build-bpf.sh index 41fc8273..9460391d 100755 --- a/scripts/build-bpf.sh +++ b/scripts/build-bpf.sh @@ -19,21 +19,12 @@ set -x # Build both parts of the program (both C and rust) via Cargo cd "${PYTH_DIR}" -cargo test --locked - # Re-run tests affected by features -cargo test --locked --features pythnet +cargo-test-bpf cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort sha256sum ./target/**/*.so -echo "Checking size of pyth_oracle.so for mainnet" -./scripts/check-size.sh 81760 -mkdir -p target/pyth/solana/ -mv target/deploy/pyth_oracle.so target/pyth/solana/pyth_oracle_solana.so - -cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort --features pythnet -sha256sum ./target/**/*.so echo "Checking size of pyth_oracle.so for pythnet" ./scripts/check-size.sh 88429 mkdir -p target/pyth/pythnet/ -mv target/deploy/pyth_oracle.so target/pyth/pythnet/pyth_oracle_pythnet.so +mv target/deploy/pyth_oracle.so target/pyth/pythnet/pyth_oracle_pythnet.so \ No newline at end of file From 05b55194ac3611d4e292570a3fe333faa19ea130 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 14:50:00 +0900 Subject: [PATCH 27/60] update c format --- .clang-format | 8 + program/c/src/oracle/native/upd_aggregate.c | 15 +- program/c/src/oracle/oracle.h | 765 ++++++++++---------- program/c/src/oracle/upd_aggregate.c | 14 +- program/c/src/oracle/upd_aggregate.h | 402 +++++----- 5 files changed, 598 insertions(+), 606 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..63a4b87f --- /dev/null +++ b/.clang-format @@ -0,0 +1,8 @@ +BasedOnStyle: LLVM +IndentWidth: 2 +AlignConsecutiveAssignments: true +AlignConsecutiveMacros: true +AlignConsecutiveDeclarations: true +BreakBeforeBraces: Linux +AllowShortIfStatementsOnASingleLine: false +UseTab: ForIndentation \ No newline at end of file diff --git a/program/c/src/oracle/native/upd_aggregate.c b/program/c/src/oracle/native/upd_aggregate.c index ef6220d8..360cbae8 100644 --- a/program/c/src/oracle/native/upd_aggregate.c +++ b/program/c/src/oracle/native/upd_aggregate.c @@ -1,7 +1,9 @@ -/// The goal of this file is to provide the upd_aggregate function to local rust tests +/// The goal of this file is to provide the upd_aggregate function to local rust +/// tests /// We need to allocate some heap space for upd_aggregate -/// When compiling for the solana runtime, the heap space is preallocated and PC_HEAP_START is provided by +/// When compiling for the solana runtime, the heap space is preallocated and +/// PC_HEAP_START is provided by char heap_start[8192]; #define PC_HEAP_START (heap_start) #define static_assert _Static_assert @@ -9,10 +11,13 @@ char heap_start[8192]; #include "../upd_aggregate.h" #include "../features.h" -extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ - return upd_aggregate(ptr, slot, timestamp ); +extern bool c_upd_aggregate_pythnet(pc_price_t *ptr, uint64_t slot, + int64_t timestamp) +{ + return upd_aggregate(ptr, slot, timestamp); } -extern void c_upd_twap( pc_price_t *ptr, int64_t nslots ){ +extern void c_upd_twap(pc_price_t *ptr, int64_t nslots) +{ upd_twap(ptr, nslots); } diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index 5146b87c..4a32eff0 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -1,12 +1,11 @@ #pragma once -#include -#include "util/compat_stdint.h" #include "features.h" +#include "util/compat_stdint.h" +#include #ifdef __cplusplus -extern "C" -{ +extern "C" { #endif // magic number at head of account @@ -19,7 +18,7 @@ extern "C" #define PC_MAX_SEND_LATENCY 25 // various size constants -#define PC_PUBKEY_SIZE 32 +#define PC_PUBKEY_SIZE 32 #define PC_PUBKEY_SIZE_64 (PC_PUBKEY_SIZE / sizeof(uint64_t)) #define PC_MAP_TABLE_SIZE 640 @@ -27,11 +26,12 @@ extern "C" #define PC_NUM_COMP_PYTHNET 127 // PC_NUM_COMP - number of price components in use -// Not whole PC_NUM_COMP_PYTHNET because of stack issues appearing in upd_aggregate() +// Not whole PC_NUM_COMP_PYTHNET because of stack issues appearing in +// upd_aggregate() #define PC_NUM_COMP 64 #define PC_PROD_ACC_SIZE 512 -#define PC_EXP_DECAY -9 +#define PC_EXP_DECAY -9 #ifndef PC_HEAP_START #define PC_HEAP_START (0x300000000) @@ -39,172 +39,166 @@ extern "C" // price types #define PC_PTYPE_UNKNOWN 0 -#define PC_PTYPE_PRICE 1 +#define PC_PTYPE_PRICE 1 // symbol status #define PC_STATUS_UNKNOWN 0 #define PC_STATUS_TRADING 1 -#define PC_STATUS_HALTED 2 +#define PC_STATUS_HALTED 2 #define PC_STATUS_AUCTION 3 #define PC_STATUS_IGNORED 4 // account types -#define PC_ACCTYPE_MAPPING 1 -#define PC_ACCTYPE_PRODUCT 2 -#define PC_ACCTYPE_PRICE 3 -#define PC_ACCTYPE_TEST 4 +#define PC_ACCTYPE_MAPPING 1 +#define PC_ACCTYPE_PRODUCT 2 +#define PC_ACCTYPE_PRICE 3 +#define PC_ACCTYPE_TEST 4 #define PC_ACCTYPE_PERMISSIONS 5 // Compute budget requested per price update instruction -// The biggest instruction appears to be about ~10300 CUs, so overestimate by 100%. +// The biggest instruction appears to be about ~10300 CUs, so overestimate by +// 100%. #define CU_BUDGET_PER_IX 20000 - // binary version of sysvar_clock account id - const uint64_t sysvar_clock[] = { - 0xc974c71817d5a706UL, - 0xb65e1d6998635628UL, - 0x5c6d4b9ba3b85e8bUL, - 0x215b5573UL}; - - // compute budget program id in hex (but wrong endianness) - /* - 321721e56f460603 - e79bc372baadecff - 6b12f7c5bbe58cbc - 403a9b432c - */ - - // correct - const uint64_t compute_budget_program_id[] = { - 0x321721e56f460603UL, - 0xe79bc372baadecffUL, - 0x6b12f7c5bbe58cbcUL, - 0x403a9b432cUL}; - - // public key of symbol or publisher account - typedef union pc_pub_key - { - uint8_t k1_[PC_PUBKEY_SIZE]; - uint64_t k8_[PC_PUBKEY_SIZE_64]; - } pc_pub_key_t; - - static_assert(sizeof(pc_pub_key_t) == 32, ""); - - // account header information - typedef struct pc_acc - { - uint32_t magic_; // pyth magic number - uint32_t ver_; // program/account version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - } pc_acc_t; - - static_assert(sizeof(pc_acc_t) == 16, ""); - - // hash table of symbol to price account mappings - typedef struct pc_map_table - { - uint32_t magic_; // pyth magic number - uint32_t ver_; // program/account version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - uint32_t num_; // number of symbols - uint32_t unused_; // 64bit padding - pc_pub_key_t next_; // next mapping account in chain - pc_pub_key_t prod_[PC_MAP_TABLE_SIZE]; // product accounts - } pc_map_table_t; - - static_assert(sizeof(pc_map_table_t) == 20536, ""); - - // variable length string - typedef struct pc_str - { - uint8_t len_; - char data_[]; - } pc_str_t; - - static_assert(sizeof(pc_str_t) == 1, ""); - - // product reference data - typedef struct pc_prod - { - uint32_t magic_; - uint32_t ver_; // program version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - pc_pub_key_t px_acc_; // first price (or quote) account - // variable number of reference key/value attribute pairs - // stored as strings (pc_str_t) - } pc_prod_t; - - static_assert(sizeof(pc_prod_t) == 48, ""); - - // price et al. for some component or aggregate - typedef struct pc_price_info - { - int64_t price_; // price per ptype_ - uint64_t conf_; // price confidence interval - uint32_t status_; // symbol status as of last update - uint32_t corp_act_status_; // corp action status as of last update - uint64_t pub_slot_; // publish slot of price - } pc_price_info_t; - - static_assert(sizeof(pc_price_info_t) == 32, ""); - - // published component price for contributing provider - typedef struct pc_price_comp - { - pc_pub_key_t pub_; // publishing key of component price - pc_price_info_t agg_; // price used in aggregate calc - pc_price_info_t latest_; // latest contributed prices - } pc_price_comp_t; - - static_assert(sizeof(pc_price_comp_t) == 96, ""); - - // time-weighted exponential moving average - typedef struct pc_ema - { - int64_t val_; // current value of ema - int64_t numer_; // numerator at full precision - int64_t denom_; // denominator at full precision - } pc_ema_t; - - static_assert(sizeof(pc_ema_t) == 24, ""); - - // price account containing aggregate and all component prices - typedef struct pc_price - { - uint32_t magic_; // pyth magic number - uint32_t ver_; // program version - uint32_t type_; // account type - uint32_t size_; // price account size - uint32_t ptype_; // price or calculation type - int32_t expo_; // price exponent - uint32_t num_; // number of component prices - uint32_t num_qt_; // number of quoters that make up aggregate - uint64_t last_slot_; // slot of last valid aggregate price - uint64_t valid_slot_; // valid on-chain slot of agg. price - pc_ema_t twap_; // time-weighted average price - pc_ema_t twac_; // time-weighted average conf interval - int64_t timestamp_; // unix timestamp of aggregate price - uint8_t min_pub_; // min publishers for valid price - int8_t message_sent_; // flag to indicate if the current aggregate price has been sent as a message to the message buffer, 0 if not sent, 1 if sent - uint8_t max_latency_; // configurable max latency in slots between send and receive - int8_t drv3_; // space for future derived values - int32_t drv4_; // space for future derived values - pc_pub_key_t prod_; // product id/ref-account - pc_pub_key_t next_; // next price account in list - uint64_t prev_slot_; // valid slot of previous aggregate with TRADING status - int64_t prev_price_; // aggregate price of previous aggregate with TRADING status - uint64_t prev_conf_; // confidence interval of previous aggregate with TRADING status - int64_t prev_timestamp_; // unix timestamp of previous aggregate with TRADING status - pc_price_info_t agg_; // aggregate price information - pc_price_comp_t comp_[PC_NUM_COMP]; // component prices - } pc_price_t; +// binary version of sysvar_clock account id +const uint64_t sysvar_clock[] = {0xc974c71817d5a706UL, 0xb65e1d6998635628UL, + 0x5c6d4b9ba3b85e8bUL, 0x215b5573UL}; +// compute budget program id in hex (but wrong endianness) /* -The value 240 is derived from the fixed size of the pc_price_t struct excluding the size of the comp_ array. -Here is the breakdown of the sizes (in bytes) for each field within the pc_price_t struct: +321721e56f460603 +e79bc372baadecff +6b12f7c5bbe58cbc +403a9b432c +*/ + +// correct +const uint64_t compute_budget_program_id[] = { + 0x321721e56f460603UL, 0xe79bc372baadecffUL, 0x6b12f7c5bbe58cbcUL, + 0x403a9b432cUL}; + +// public key of symbol or publisher account +typedef union pc_pub_key { + uint8_t k1_[PC_PUBKEY_SIZE]; + uint64_t k8_[PC_PUBKEY_SIZE_64]; +} pc_pub_key_t; + +static_assert(sizeof(pc_pub_key_t) == 32, ""); + +// account header information +typedef struct pc_acc { + uint32_t magic_; // pyth magic number + uint32_t ver_; // program/account version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account +} pc_acc_t; + +static_assert(sizeof(pc_acc_t) == 16, ""); + +// hash table of symbol to price account mappings +typedef struct pc_map_table { + uint32_t magic_; // pyth magic number + uint32_t ver_; // program/account version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + uint32_t num_; // number of symbols + uint32_t unused_; // 64bit padding + pc_pub_key_t next_; // next mapping account in chain + pc_pub_key_t prod_[PC_MAP_TABLE_SIZE]; // product accounts +} pc_map_table_t; + +static_assert(sizeof(pc_map_table_t) == 20536, ""); + +// variable length string +typedef struct pc_str { + uint8_t len_; + char data_[]; +} pc_str_t; + +static_assert(sizeof(pc_str_t) == 1, ""); + +// product reference data +typedef struct pc_prod { + uint32_t magic_; + uint32_t ver_; // program version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + pc_pub_key_t px_acc_; // first price (or quote) account + // variable number of reference key/value attribute pairs + // stored as strings (pc_str_t) +} pc_prod_t; + +static_assert(sizeof(pc_prod_t) == 48, ""); + +// price et al. for some component or aggregate +typedef struct pc_price_info { + int64_t price_; // price per ptype_ + uint64_t conf_; // price confidence interval + uint32_t status_; // symbol status as of last update + uint32_t corp_act_status_; // corp action status as of last update + uint64_t pub_slot_; // publish slot of price +} pc_price_info_t; + +static_assert(sizeof(pc_price_info_t) == 32, ""); + +// published component price for contributing provider +typedef struct pc_price_comp { + pc_pub_key_t pub_; // publishing key of component price + pc_price_info_t agg_; // price used in aggregate calc + pc_price_info_t latest_; // latest contributed prices +} pc_price_comp_t; + +static_assert(sizeof(pc_price_comp_t) == 96, ""); + +// time-weighted exponential moving average +typedef struct pc_ema { + int64_t val_; // current value of ema + int64_t numer_; // numerator at full precision + int64_t denom_; // denominator at full precision +} pc_ema_t; + +static_assert(sizeof(pc_ema_t) == 24, ""); + +// price account containing aggregate and all component prices +typedef struct pc_price { + uint32_t magic_; // pyth magic number + uint32_t ver_; // program version + uint32_t type_; // account type + uint32_t size_; // price account size + uint32_t ptype_; // price or calculation type + int32_t expo_; // price exponent + uint32_t num_; // number of component prices + uint32_t num_qt_; // number of quoters that make up aggregate + uint64_t last_slot_; // slot of last valid aggregate price + uint64_t valid_slot_; // valid on-chain slot of agg. price + pc_ema_t twap_; // time-weighted average price + pc_ema_t twac_; // time-weighted average conf interval + int64_t timestamp_; // unix timestamp of aggregate price + uint8_t min_pub_; // min publishers for valid price + int8_t message_sent_; // flag to indicate if the current aggregate price has + // been sent as a message to the message buffer, 0 if + // not sent, 1 if sent + uint8_t max_latency_; // configurable max latency in slots between send and + // receive + int8_t drv3_; // space for future derived values + int32_t drv4_; // space for future derived values + pc_pub_key_t prod_; // product id/ref-account + pc_pub_key_t next_; // next price account in list + uint64_t prev_slot_; // valid slot of previous aggregate with TRADING status + int64_t prev_price_; // aggregate price of previous aggregate with TRADING + // status + uint64_t prev_conf_; // confidence interval of previous aggregate with + // TRADING status + int64_t prev_timestamp_; // unix timestamp of previous aggregate with + // TRADING status + pc_price_info_t agg_; // aggregate price information + pc_price_comp_t comp_[PC_NUM_COMP]; // component prices +} pc_price_t; + +/* +The value 240 is derived from the fixed size of the pc_price_t struct excluding +the size of the comp_ array. Here is the breakdown of the sizes (in bytes) for +each field within the pc_price_t struct: - magic_ (uint32_t): 4 bytes - ver_ (uint32_t): 4 bytes @@ -224,7 +218,8 @@ Here is the breakdown of the sizes (in bytes) for each field within the pc_price - max_latency_ (uint8_t): 1 byte - drv3_ (int8_t): 1 byte - drv4_ (int32_t): 4 bytes -- prod_ (pc_pub_key_t): 32 bytes (assuming pc_pub_key_t is a 32-byte array or struct) +- prod_ (pc_pub_key_t): 32 bytes (assuming pc_pub_key_t is a 32-byte array or +struct) - next_ (pc_pub_key_t): 32 bytes (similar to prod_) - prev_slot_ (uint64_t): 8 bytes - prev_price_ (int64_t): 8 bytes @@ -232,242 +227,230 @@ Here is the breakdown of the sizes (in bytes) for each field within the pc_price - prev_timestamp_ (int64_t): 8 bytes - agg_ (pc_price_info_t): 32 bytes -Adding up all these sizes gives us a total of 240 bytes for the fixed part of the pc_price_t struct. -The size of the comp_ array is variable and depends on PC_NUM_COMP, hence it is calculated separately and added to the base size of 240 bytes. +Adding up all these sizes gives us a total of 240 bytes for the fixed part of +the pc_price_t struct. The size of the comp_ array is variable and depends on +PC_NUM_COMP, hence it is calculated separately and added to the base size of 240 +bytes. */ -#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (240 + PC_NUM_COMP * sizeof(pc_price_comp_t)) +#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET \ + (240 + PC_NUM_COMP * sizeof(pc_price_comp_t)) - static_assert(sizeof(pc_price_t) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, ""); +static_assert(sizeof(pc_price_t) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, ""); #undef PC_EXPECTED_PRICE_T_SIZE_PYTHNET - // This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd. - // It is set tighly to the current price account + 96 component prices + 48 bytes for cumulative sums - const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof(pc_price_comp_t) + 48; - - // command enumeration - typedef enum - { - - // initialize first mapping list account - // key[0] funding account [signer writable] - // key[1] mapping account [signer writable] - e_cmd_init_mapping = 0, - - // initialize and add new mapping account - // key[0] funding account [signer writable] - // key[1] tail mapping account [signer writable] - // key[2] new mapping account [signer writable] - e_cmd_add_mapping, - - // initialize and add new product reference data account - // key[0] funding account [signer writable] - // key[1] mapping account [signer writable] - // key[2] new product account [signer writable] - e_cmd_add_product, - - // update product account - // key[0] funding account [signer writable] - // key[1] product account [signer writable] - e_cmd_upd_product, - - // add new price account to a product account - // key[0] funding account [signer writable] - // key[1] product account [signer writable] - // key[2] new price account [signer writable] - e_cmd_add_price, - - // add publisher to symbol account - // key[0] funding account [signer writable] - // key[1] price account [signer writable] - e_cmd_add_publisher, - - // delete publisher from symbol account - // key[0] funding account [signer writable] - // key[1] price account [signer writable] - e_cmd_del_publisher, - - // publish component price - // key[0] funding account [signer writable] - // key[1] price account [writable] - // key[2] sysvar_clock account [readable] - e_cmd_upd_price, - - // compute aggregate price - // key[0] funding account [signer writable] - // key[1] price account [writable] - // key[2] sysvar_clock account [readable] - e_cmd_agg_price, - - // (re)initialize price account - // key[0] funding account [signer writable] - // key[1] new price account [signer writable] - e_cmd_init_price, - - // deprecated - e_cmd_init_test, - - // deprecated - e_cmd_upd_test, - - // set min publishers - // key[0] funding account [signer writable] - // key[1] price account [signer writable] - e_cmd_set_min_pub, - - // publish component price, never returning an error even if the update failed - // key[0] funding account [signer writable] - // key[1] price account [writable] - // key[2] sysvar_clock account [readable] - e_cmd_upd_price_no_fail_on_error, - - // resizes a price account so that it fits the Time Machine - // key[0] funding account [signer writable] - // key[1] price account [Signer writable] - // key[2] system program [readable] - e_cmd_resize_price_account, - - // deletes a price account - // key[0] funding account [signer writable] - // key[1] product account [signer writable] - // key[2] price account [signer writable] - e_cmd_del_price, - - // deletes a product account - // key[0] funding account [signer writable] - // key[1] mapping account [signer writable] - // key[2] product account [signer writable] - e_cmd_del_product, - } command_t; - - typedef struct cmd_hdr - { - uint32_t ver_; - int32_t cmd_; - } cmd_hdr_t; - - static_assert(sizeof(cmd_hdr_t) == 8, ""); - - typedef struct cmd_add_product - { - uint32_t ver_; - int32_t cmd_; - } cmd_add_product_t; - - static_assert(sizeof(cmd_add_product_t) == 8, ""); - - typedef struct cmd_upd_product - { - uint32_t ver_; - int32_t cmd_; - // set of key-value pairs - } cmd_upd_product_t; - - static_assert(sizeof(cmd_upd_product_t) == 8, ""); - - typedef struct cmd_add_price - { - uint32_t ver_; - int32_t cmd_; - int32_t expo_; - uint32_t ptype_; - } cmd_add_price_t; - - static_assert(sizeof(cmd_add_price_t) == 16, ""); - - typedef struct cmd_init_price - { - uint32_t ver_; - int32_t cmd_; - int32_t expo_; - uint32_t ptype_; - } cmd_init_price_t; - - static_assert(sizeof(cmd_init_price_t) == 16, ""); - - typedef struct cmd_set_min_pub - { - uint32_t ver_; - int32_t cmd_; - uint8_t min_pub_; - } cmd_set_min_pub_t; - - static_assert(sizeof(cmd_set_min_pub_t) == 12, ""); - - typedef struct cmd_add_publisher - { - uint32_t ver_; - int32_t cmd_; - pc_pub_key_t pub_; - } cmd_add_publisher_t; - - static_assert(sizeof(cmd_add_publisher_t) == 40, ""); - - typedef struct cmd_del_publisher - { - uint32_t ver_; - int32_t cmd_; - pc_pub_key_t pub_; - } cmd_del_publisher_t; - - static_assert(sizeof(cmd_del_publisher_t) == 40, ""); - - typedef struct cmd_upd_price - { - uint32_t ver_; - int32_t cmd_; - uint32_t status_; - uint32_t unused_; - int64_t price_; - uint64_t conf_; - uint64_t pub_slot_; - } cmd_upd_price_t; - - static_assert(sizeof(cmd_upd_price_t) == 40, ""); - - // structure of clock sysvar account - typedef struct sysvar_clock - { - uint64_t slot_; - int64_t epoch_start_timestamp_; - uint64_t epoch_; - uint64_t leader_schedule_epoch_; - int64_t unix_timestamp_; - } sysvar_clock_t; - - static_assert(sizeof(sysvar_clock_t) == 40, ""); - - // compare if two pub_keys (accounts) are the same - inline bool pc_pub_key_equal(pc_pub_key_t *p1, pc_pub_key_t *p2) - { - return p1->k8_[0] == p2->k8_[0] && - p1->k8_[1] == p2->k8_[1] && - p1->k8_[2] == p2->k8_[2] && - p1->k8_[3] == p2->k8_[3]; - } - - // check for null (zero) public key - inline bool pc_pub_key_is_zero(pc_pub_key_t *p) - { - return p->k8_[0] == 0UL && - p->k8_[1] == 0UL && - p->k8_[2] == 0UL && - p->k8_[3] == 0UL; - } - - // set public key to zero - inline void pc_pub_key_set_zero(pc_pub_key_t *p) - { - p->k8_[0] = p->k8_[1] = p->k8_[2] = p->k8_[3] = 0UL; - } - - // assign one pub_key from another - inline void pc_pub_key_assign(pc_pub_key_t *tgt, pc_pub_key_t *src) - { - tgt->k8_[0] = src->k8_[0]; - tgt->k8_[1] = src->k8_[1]; - tgt->k8_[2] = src->k8_[2]; - tgt->k8_[3] = src->k8_[3]; - } +// This constant needs to be an upper bound of the price account size, it is +// used within pythd for ztsd. It is set tighly to the current price account + +// 96 component prices + 48 bytes for cumulative sums +const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof(pc_price_comp_t) + 48; + +// command enumeration +typedef enum { + + // initialize first mapping list account + // key[0] funding account [signer writable] + // key[1] mapping account [signer writable] + e_cmd_init_mapping = 0, + + // initialize and add new mapping account + // key[0] funding account [signer writable] + // key[1] tail mapping account [signer writable] + // key[2] new mapping account [signer writable] + e_cmd_add_mapping, + + // initialize and add new product reference data account + // key[0] funding account [signer writable] + // key[1] mapping account [signer writable] + // key[2] new product account [signer writable] + e_cmd_add_product, + + // update product account + // key[0] funding account [signer writable] + // key[1] product account [signer writable] + e_cmd_upd_product, + + // add new price account to a product account + // key[0] funding account [signer writable] + // key[1] product account [signer writable] + // key[2] new price account [signer writable] + e_cmd_add_price, + + // add publisher to symbol account + // key[0] funding account [signer writable] + // key[1] price account [signer writable] + e_cmd_add_publisher, + + // delete publisher from symbol account + // key[0] funding account [signer writable] + // key[1] price account [signer writable] + e_cmd_del_publisher, + + // publish component price + // key[0] funding account [signer writable] + // key[1] price account [writable] + // key[2] sysvar_clock account [readable] + e_cmd_upd_price, + + // compute aggregate price + // key[0] funding account [signer writable] + // key[1] price account [writable] + // key[2] sysvar_clock account [readable] + e_cmd_agg_price, + + // (re)initialize price account + // key[0] funding account [signer writable] + // key[1] new price account [signer writable] + e_cmd_init_price, + + // deprecated + e_cmd_init_test, + + // deprecated + e_cmd_upd_test, + + // set min publishers + // key[0] funding account [signer writable] + // key[1] price account [signer writable] + e_cmd_set_min_pub, + + // publish component price, never returning an error even if the update + // failed key[0] funding account [signer writable] key[1] price + // account [writable] key[2] sysvar_clock account [readable] + e_cmd_upd_price_no_fail_on_error, + + // resizes a price account so that it fits the Time Machine + // key[0] funding account [signer writable] + // key[1] price account [Signer writable] + // key[2] system program [readable] + e_cmd_resize_price_account, + + // deletes a price account + // key[0] funding account [signer writable] + // key[1] product account [signer writable] + // key[2] price account [signer writable] + e_cmd_del_price, + + // deletes a product account + // key[0] funding account [signer writable] + // key[1] mapping account [signer writable] + // key[2] product account [signer writable] + e_cmd_del_product, +} command_t; + +typedef struct cmd_hdr { + uint32_t ver_; + int32_t cmd_; +} cmd_hdr_t; + +static_assert(sizeof(cmd_hdr_t) == 8, ""); + +typedef struct cmd_add_product { + uint32_t ver_; + int32_t cmd_; +} cmd_add_product_t; + +static_assert(sizeof(cmd_add_product_t) == 8, ""); + +typedef struct cmd_upd_product { + uint32_t ver_; + int32_t cmd_; + // set of key-value pairs +} cmd_upd_product_t; + +static_assert(sizeof(cmd_upd_product_t) == 8, ""); + +typedef struct cmd_add_price { + uint32_t ver_; + int32_t cmd_; + int32_t expo_; + uint32_t ptype_; +} cmd_add_price_t; + +static_assert(sizeof(cmd_add_price_t) == 16, ""); + +typedef struct cmd_init_price { + uint32_t ver_; + int32_t cmd_; + int32_t expo_; + uint32_t ptype_; +} cmd_init_price_t; + +static_assert(sizeof(cmd_init_price_t) == 16, ""); + +typedef struct cmd_set_min_pub { + uint32_t ver_; + int32_t cmd_; + uint8_t min_pub_; +} cmd_set_min_pub_t; + +static_assert(sizeof(cmd_set_min_pub_t) == 12, ""); + +typedef struct cmd_add_publisher { + uint32_t ver_; + int32_t cmd_; + pc_pub_key_t pub_; +} cmd_add_publisher_t; + +static_assert(sizeof(cmd_add_publisher_t) == 40, ""); + +typedef struct cmd_del_publisher { + uint32_t ver_; + int32_t cmd_; + pc_pub_key_t pub_; +} cmd_del_publisher_t; + +static_assert(sizeof(cmd_del_publisher_t) == 40, ""); + +typedef struct cmd_upd_price { + uint32_t ver_; + int32_t cmd_; + uint32_t status_; + uint32_t unused_; + int64_t price_; + uint64_t conf_; + uint64_t pub_slot_; +} cmd_upd_price_t; + +static_assert(sizeof(cmd_upd_price_t) == 40, ""); + +// structure of clock sysvar account +typedef struct sysvar_clock { + uint64_t slot_; + int64_t epoch_start_timestamp_; + uint64_t epoch_; + uint64_t leader_schedule_epoch_; + int64_t unix_timestamp_; +} sysvar_clock_t; + +static_assert(sizeof(sysvar_clock_t) == 40, ""); + +// compare if two pub_keys (accounts) are the same +inline bool pc_pub_key_equal(pc_pub_key_t *p1, pc_pub_key_t *p2) +{ + return p1->k8_[0] == p2->k8_[0] && p1->k8_[1] == p2->k8_[1] && + p1->k8_[2] == p2->k8_[2] && p1->k8_[3] == p2->k8_[3]; +} + +// check for null (zero) public key +inline bool pc_pub_key_is_zero(pc_pub_key_t *p) +{ + return p->k8_[0] == 0UL && p->k8_[1] == 0UL && p->k8_[2] == 0UL && + p->k8_[3] == 0UL; +} + +// set public key to zero +inline void pc_pub_key_set_zero(pc_pub_key_t *p) +{ + p->k8_[0] = p->k8_[1] = p->k8_[2] = p->k8_[3] = 0UL; +} + +// assign one pub_key from another +inline void pc_pub_key_assign(pc_pub_key_t *tgt, pc_pub_key_t *src) +{ + tgt->k8_[0] = src->k8_[0]; + tgt->k8_[1] = src->k8_[1]; + tgt->k8_[2] = src->k8_[2]; + tgt->k8_[3] = src->k8_[3]; +} #ifdef __cplusplus } diff --git a/program/c/src/oracle/upd_aggregate.c b/program/c/src/oracle/upd_aggregate.c index 0c723a69..8c064b03 100644 --- a/program/c/src/oracle/upd_aggregate.c +++ b/program/c/src/oracle/upd_aggregate.c @@ -1,16 +1,18 @@ /* * BPF upd_aggregate binding */ -#include -#include "oracle.h" #include "upd_aggregate.h" #include "features.h" +#include "oracle.h" +#include - -extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ - return upd_aggregate(ptr, slot, timestamp ); +extern bool c_upd_aggregate_pythnet(pc_price_t *ptr, uint64_t slot, + int64_t timestamp) +{ + return upd_aggregate(ptr, slot, timestamp); } -extern void c_upd_twap( pc_price_t *ptr, int64_t nslots ){ +extern void c_upd_twap(pc_price_t *ptr, int64_t nslots) +{ upd_twap(ptr, nslots); } diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index e5e125b6..a63190d9 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -1,228 +1,222 @@ #pragma once -#include "oracle.h" -#include "model/price_model.h" #include "model/price_model.c" /* FIXME: HACK TO DEAL WITH DOCKER LINKAGE ISSUES */ +#include "model/price_model.h" +#include "oracle.h" #include "pd.h" #ifdef __cplusplus -extern "C" -{ +extern "C" { #endif - typedef struct pc_qset - { - pd_t iprice_[PC_NUM_COMP]; - pd_t uprice_[PC_NUM_COMP]; - pd_t lprice_[PC_NUM_COMP]; - pd_t weight_[PC_NUM_COMP]; - int64_t decay_[1 + PC_MAX_SEND_LATENCY]; - int64_t fact_[PC_FACTOR_SIZE]; - } pc_qset_t; - - // initialize quote-set temporary data in heap area - static pc_qset_t *qset_new() - { - // allocate off heap - pc_qset_t *qs = (pc_qset_t *)PC_HEAP_START; - - // sqrt of numbers 1 to 25 for decaying conf. interval based on slot delay - qs->decay_[0] = 1000000000L; - qs->decay_[1] = 1000000000L; - qs->decay_[2] = 1414213562L; - qs->decay_[3] = 1732050807L; - qs->decay_[4] = 2000000000L; - qs->decay_[5] = 2236067977L; - qs->decay_[6] = 2449489742L; - qs->decay_[7] = 2645751311L; - qs->decay_[8] = 2828427124L; - qs->decay_[9] = 3000000000L; - qs->decay_[10] = 3162277660L; - qs->decay_[11] = 3316624790L; - qs->decay_[12] = 3464101615L; - qs->decay_[13] = 3605551275L; - qs->decay_[14] = 3741657386L; - qs->decay_[15] = 3872983346L; - qs->decay_[16] = 4000000000L; - qs->decay_[17] = 4123105625L; - qs->decay_[18] = 4242640687L; - qs->decay_[19] = 4358898943L; - qs->decay_[20] = 4472135954L; - qs->decay_[21] = 4582575694L; - qs->decay_[22] = 4690415759L; - qs->decay_[23] = 4795831523L; - qs->decay_[24] = 4898979485L; - qs->decay_[25] = 5000000000L; - - // powers of 10 for use in decimal arithmetic scaling - qs->fact_[0] = 1L; - qs->fact_[1] = 10L; - qs->fact_[2] = 100L; - qs->fact_[3] = 1000L; - qs->fact_[4] = 10000L; - qs->fact_[5] = 100000L; - qs->fact_[6] = 1000000L; - qs->fact_[7] = 10000000L; - qs->fact_[8] = 100000000L; - qs->fact_[9] = 1000000000L; - qs->fact_[10] = 10000000000L; - qs->fact_[11] = 100000000000L; - qs->fact_[12] = 1000000000000L; - qs->fact_[13] = 10000000000000L; - qs->fact_[14] = 100000000000000L; - qs->fact_[15] = 1000000000000000L; - qs->fact_[16] = 10000000000000000L; - qs->fact_[17] = 100000000000000000L; - - return qs; - } - - static void upd_ema( - pc_ema_t *ptr, pd_t *val, pd_t *conf, int64_t nslot, pc_qset_t *qs, int32_t expo) - { - pd_t numer[1], denom[1], cwgt[1], wval[1], decay[1], diff[1], one[1]; - pd_new(one, 100000000L, -8); - if (conf->v_) - { - pd_div(cwgt, one, conf); - } - else - { - pd_set(cwgt, one); - } - if (nslot > PD_EMA_MAX_DIFF) - { - // initial condition - pd_mul(numer, val, cwgt); - pd_set(denom, cwgt); - } - else - { - // compute decay factor - pd_new(diff, nslot, 0); - pd_new(decay, PD_EMA_DECAY, PD_EMA_EXPO); - pd_mul(decay, decay, diff); - pd_add(decay, decay, one, qs->fact_); - - // compute numer/denom and new value from decay factor - pd_load(numer, ptr->numer_); - pd_load(denom, ptr->denom_); - pd_mul(numer, numer, decay); - pd_mul(wval, val, cwgt); - pd_add(numer, numer, wval, qs->fact_); - pd_mul(denom, denom, decay); - pd_add(denom, denom, cwgt, qs->fact_); - pd_div(val, numer, denom); - } - +typedef struct pc_qset { + pd_t iprice_[PC_NUM_COMP]; + pd_t uprice_[PC_NUM_COMP]; + pd_t lprice_[PC_NUM_COMP]; + pd_t weight_[PC_NUM_COMP]; + int64_t decay_[1 + PC_MAX_SEND_LATENCY]; + int64_t fact_[PC_FACTOR_SIZE]; +} pc_qset_t; + +// initialize quote-set temporary data in heap area +static pc_qset_t *qset_new() +{ + // allocate off heap + pc_qset_t *qs = (pc_qset_t *)PC_HEAP_START; + + // sqrt of numbers 1 to 25 for decaying conf. interval based on slot delay + qs->decay_[0] = 1000000000L; + qs->decay_[1] = 1000000000L; + qs->decay_[2] = 1414213562L; + qs->decay_[3] = 1732050807L; + qs->decay_[4] = 2000000000L; + qs->decay_[5] = 2236067977L; + qs->decay_[6] = 2449489742L; + qs->decay_[7] = 2645751311L; + qs->decay_[8] = 2828427124L; + qs->decay_[9] = 3000000000L; + qs->decay_[10] = 3162277660L; + qs->decay_[11] = 3316624790L; + qs->decay_[12] = 3464101615L; + qs->decay_[13] = 3605551275L; + qs->decay_[14] = 3741657386L; + qs->decay_[15] = 3872983346L; + qs->decay_[16] = 4000000000L; + qs->decay_[17] = 4123105625L; + qs->decay_[18] = 4242640687L; + qs->decay_[19] = 4358898943L; + qs->decay_[20] = 4472135954L; + qs->decay_[21] = 4582575694L; + qs->decay_[22] = 4690415759L; + qs->decay_[23] = 4795831523L; + qs->decay_[24] = 4898979485L; + qs->decay_[25] = 5000000000L; + + // powers of 10 for use in decimal arithmetic scaling + qs->fact_[0] = 1L; + qs->fact_[1] = 10L; + qs->fact_[2] = 100L; + qs->fact_[3] = 1000L; + qs->fact_[4] = 10000L; + qs->fact_[5] = 100000L; + qs->fact_[6] = 1000000L; + qs->fact_[7] = 10000000L; + qs->fact_[8] = 100000000L; + qs->fact_[9] = 1000000000L; + qs->fact_[10] = 10000000000L; + qs->fact_[11] = 100000000000L; + qs->fact_[12] = 1000000000000L; + qs->fact_[13] = 10000000000000L; + qs->fact_[14] = 100000000000000L; + qs->fact_[15] = 1000000000000000L; + qs->fact_[16] = 10000000000000000L; + qs->fact_[17] = 100000000000000000L; + + return qs; +} - // adjust and store results - pd_adjust(val, expo, qs->fact_); - ptr->val_ = val->v_; - int64_t numer1, denom1; - if (pd_store(&numer1, numer) && pd_store(&denom1, denom)) - { - ptr->numer_ = numer1; - ptr->denom_ = denom1; - } +static void upd_ema(pc_ema_t *ptr, pd_t *val, pd_t *conf, int64_t nslot, + pc_qset_t *qs, int32_t expo) +{ + pd_t numer[1], denom[1], cwgt[1], wval[1], decay[1], diff[1], one[1]; + pd_new(one, 100000000L, -8); + if (conf->v_) { + pd_div(cwgt, one, conf); + } else { + pd_set(cwgt, one); + } + if (nslot > PD_EMA_MAX_DIFF) { + // initial condition + pd_mul(numer, val, cwgt); + pd_set(denom, cwgt); + } else { + // compute decay factor + pd_new(diff, nslot, 0); + pd_new(decay, PD_EMA_DECAY, PD_EMA_EXPO); + pd_mul(decay, decay, diff); + pd_add(decay, decay, one, qs->fact_); + + // compute numer/denom and new value from decay factor + pd_load(numer, ptr->numer_); + pd_load(denom, ptr->denom_); + pd_mul(numer, numer, decay); + pd_mul(wval, val, cwgt); + pd_add(numer, numer, wval, qs->fact_); + pd_mul(denom, denom, decay); + pd_add(denom, denom, cwgt, qs->fact_); + pd_div(val, numer, denom); } - static inline void upd_twap( - pc_price_t *ptr, int64_t nslots) - { - pc_qset_t *qs = qset_new(); - - pd_t px[1], conf[1]; - pd_new_scale(px, ptr->agg_.price_, ptr->expo_); - pd_new_scale(conf, (int64_t)(ptr->agg_.conf_), ptr->expo_); - upd_ema(&ptr->twap_, px, conf, nslots, qs, ptr->expo_); - upd_ema(&ptr->twac_, conf, conf, nslots, qs, ptr->expo_); + // adjust and store results + pd_adjust(val, expo, qs->fact_); + ptr->val_ = val->v_; + int64_t numer1, denom1; + if (pd_store(&numer1, numer) && pd_store(&denom1, denom)) { + ptr->numer_ = numer1; + ptr->denom_ = denom1; } +} - // update aggregate price - static inline bool upd_aggregate(pc_price_t *ptr, uint64_t slot, int64_t timestamp) - { - // update aggregate details ready for next slot - ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price - ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price - ptr->timestamp_ = timestamp; - - // identify valid quotes - // compute the aggregate prices and ranges - int64_t agg_price; - int64_t agg_conf; - { - uint32_t numv = 0; - uint32_t nprcs = (uint32_t)0; - int64_t prcs[PC_NUM_COMP * 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) - for (uint32_t i = 0; i != ptr->num_; ++i) - { - pc_price_comp_t *iptr = &ptr->comp_[i]; - // copy contributing price to aggregate snapshot - iptr->agg_ = iptr->latest_; - // add quote to sorted permutation array if it is valid - int64_t slot_diff = (int64_t)slot - (int64_t)(iptr->agg_.pub_slot_); - int64_t price = iptr->agg_.price_; - int64_t conf = (int64_t)(iptr->agg_.conf_); - int64_t max_latency = ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY; - if (iptr->agg_.status_ == PC_STATUS_TRADING && - // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < INT64_MAX - // These checks ensure that price - conf and price + conf do not overflow. - (int64_t)0 < conf && (INT64_MIN + conf) <= price && price <= (INT64_MAX - conf) && - // slot_diff is implicitly >= 0 due to the check in Rust code ensuring publishing_slot is always less than or equal to the current slot. - slot_diff <= max_latency) - { - numv += 1; - prcs[nprcs++] = price - conf; - prcs[nprcs++] = price; - prcs[nprcs++] = price + conf; - } - } +static inline void upd_twap(pc_price_t *ptr, int64_t nslots) +{ + pc_qset_t *qs = qset_new(); - // too few valid quotes - ptr->num_qt_ = numv; - if (numv == 0 || numv < ptr->min_pub_) - { - ptr->agg_.status_ = PC_STATUS_UNKNOWN; - return false; - } + pd_t px[1], conf[1]; + pd_new_scale(px, ptr->agg_.price_, ptr->expo_); + pd_new_scale(conf, (int64_t)(ptr->agg_.conf_), ptr->expo_); + upd_ema(&ptr->twap_, px, conf, nslots, qs, ptr->expo_); + upd_ema(&ptr->twac_, conf, conf, nslots, qs, ptr->expo_); +} - // evaluate the model to get the p25/p50/p75 prices - // note: numv>0 and nprcs = 3*numv at this point - int64_t agg_p25; - int64_t agg_p75; - int64_t scratch[PC_NUM_COMP * 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) - price_model_core((uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, scratch); - - // get the left and right confidences - // note that as valid quotes have positive prices currently and - // agg_p25, agg_price, agg_p75 are ordered, this calculation can't - // overflow / underflow - int64_t agg_conf_left = agg_price - agg_p25; - int64_t agg_conf_right = agg_p75 - agg_price; - - // use the larger of the left and right confidences - agg_conf = agg_conf_right > agg_conf_left ? agg_conf_right : agg_conf_left; - - // if the confidences end up at zero, we abort - // this is paranoia as it is currently not possible when nprcs>2 and - // positive confidences given the current pricing model - if (agg_conf <= (int64_t)0) - { - ptr->agg_.status_ = PC_STATUS_UNKNOWN; - return false; +// update aggregate price +static inline bool upd_aggregate(pc_price_t *ptr, uint64_t slot, + int64_t timestamp) +{ + // update aggregate details ready for next slot + ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price + ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price + ptr->timestamp_ = timestamp; + + // identify valid quotes + // compute the aggregate prices and ranges + int64_t agg_price; + int64_t agg_conf; + { + uint32_t numv = 0; + uint32_t nprcs = (uint32_t)0; + int64_t + prcs[PC_NUM_COMP * 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: + // DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + for (uint32_t i = 0; i != ptr->num_; ++i) { + pc_price_comp_t *iptr = &ptr->comp_[i]; + // copy contributing price to aggregate snapshot + iptr->agg_ = iptr->latest_; + // add quote to sorted permutation array if it is valid + int64_t slot_diff = (int64_t)slot - (int64_t)(iptr->agg_.pub_slot_); + int64_t price = iptr->agg_.price_; + int64_t conf = (int64_t)(iptr->agg_.conf_); + int64_t max_latency = + ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY; + if (iptr->agg_.status_ == PC_STATUS_TRADING && + // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < + // INT64_MAX These checks ensure that price - conf and price + conf do + // not overflow. + (int64_t)0 < conf && (INT64_MIN + conf) <= price && + price <= (INT64_MAX - conf) && + // slot_diff is implicitly >= 0 due to the check in Rust code ensuring + // publishing_slot is always less than or equal to the current slot. + slot_diff <= max_latency) { + numv += 1; + prcs[nprcs++] = price - conf; + prcs[nprcs++] = price; + prcs[nprcs++] = price + conf; } } - // update status and publish slot of last trading status price - ptr->agg_.status_ = PC_STATUS_TRADING; - ptr->last_slot_ = slot; - ptr->agg_.price_ = agg_price; - ptr->agg_.conf_ = (uint64_t)agg_conf; + // too few valid quotes + ptr->num_qt_ = numv; + if (numv == 0 || numv < ptr->min_pub_) { + ptr->agg_.status_ = PC_STATUS_UNKNOWN; + return false; + } - return true; + // evaluate the model to get the p25/p50/p75 prices + // note: numv>0 and nprcs = 3*numv at this point + int64_t agg_p25; + int64_t agg_p75; + int64_t scratch[PC_NUM_COMP * + 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK + // THIS FITS INTO STACK FRAME LIMIT) + price_model_core((uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, + scratch); + + // get the left and right confidences + // note that as valid quotes have positive prices currently and + // agg_p25, agg_price, agg_p75 are ordered, this calculation can't + // overflow / underflow + int64_t agg_conf_left = agg_price - agg_p25; + int64_t agg_conf_right = agg_p75 - agg_price; + + // use the larger of the left and right confidences + agg_conf = agg_conf_right > agg_conf_left ? agg_conf_right : agg_conf_left; + + // if the confidences end up at zero, we abort + // this is paranoia as it is currently not possible when nprcs>2 and + // positive confidences given the current pricing model + if (agg_conf <= (int64_t)0) { + ptr->agg_.status_ = PC_STATUS_UNKNOWN; + return false; + } } + // update status and publish slot of last trading status price + ptr->agg_.status_ = PC_STATUS_TRADING; + ptr->last_slot_ = slot; + ptr->agg_.price_ = agg_price; + ptr->agg_.conf_ = (uint64_t)agg_conf; + + return true; +} #ifdef __cplusplus } From 5817b1d88c6e99e6d10271442ba9e876401d7c5e Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 14:55:20 +0900 Subject: [PATCH 28/60] revert format --- program/c/src/oracle/native/upd_aggregate.c | 15 +++++---------- program/c/src/oracle/upd_aggregate.c | 14 ++++++-------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/program/c/src/oracle/native/upd_aggregate.c b/program/c/src/oracle/native/upd_aggregate.c index 360cbae8..ef6220d8 100644 --- a/program/c/src/oracle/native/upd_aggregate.c +++ b/program/c/src/oracle/native/upd_aggregate.c @@ -1,9 +1,7 @@ -/// The goal of this file is to provide the upd_aggregate function to local rust -/// tests +/// The goal of this file is to provide the upd_aggregate function to local rust tests /// We need to allocate some heap space for upd_aggregate -/// When compiling for the solana runtime, the heap space is preallocated and -/// PC_HEAP_START is provided by +/// When compiling for the solana runtime, the heap space is preallocated and PC_HEAP_START is provided by char heap_start[8192]; #define PC_HEAP_START (heap_start) #define static_assert _Static_assert @@ -11,13 +9,10 @@ char heap_start[8192]; #include "../upd_aggregate.h" #include "../features.h" -extern bool c_upd_aggregate_pythnet(pc_price_t *ptr, uint64_t slot, - int64_t timestamp) -{ - return upd_aggregate(ptr, slot, timestamp); +extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ + return upd_aggregate(ptr, slot, timestamp ); } -extern void c_upd_twap(pc_price_t *ptr, int64_t nslots) -{ +extern void c_upd_twap( pc_price_t *ptr, int64_t nslots ){ upd_twap(ptr, nslots); } diff --git a/program/c/src/oracle/upd_aggregate.c b/program/c/src/oracle/upd_aggregate.c index 8c064b03..0c723a69 100644 --- a/program/c/src/oracle/upd_aggregate.c +++ b/program/c/src/oracle/upd_aggregate.c @@ -1,18 +1,16 @@ /* * BPF upd_aggregate binding */ +#include +#include "oracle.h" #include "upd_aggregate.h" #include "features.h" -#include "oracle.h" -#include -extern bool c_upd_aggregate_pythnet(pc_price_t *ptr, uint64_t slot, - int64_t timestamp) -{ - return upd_aggregate(ptr, slot, timestamp); + +extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ + return upd_aggregate(ptr, slot, timestamp ); } -extern void c_upd_twap(pc_price_t *ptr, int64_t nslots) -{ +extern void c_upd_twap( pc_price_t *ptr, int64_t nslots ){ upd_twap(ptr, nslots); } From 72196d8484f0f6f94ebd825b517914ca5aa0fd80 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 14:56:22 +0900 Subject: [PATCH 29/60] revert format --- .github/workflows/docker.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 96452069..d05dac42 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -2,11 +2,11 @@ name: Docker on: push: - branches: [main] - tags: + branches: [ main ] + tags : - oracle-v* pull_request: - branches: [main] + branches: [ main ] env: SOLANA_VERSION: 1.14.7 # Using 1.14.x to have sbf instead of bpf. bpf does not work anymore. @@ -19,6 +19,7 @@ env: IS_ORACLE_RELEASE: ${{ startsWith( github.ref, 'refs/tags/oracle-' ) }} + jobs: build: runs-on: ubuntu-latest From dc91c22084ab22e74d33290704d4c11289db553c Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 14:57:20 +0900 Subject: [PATCH 30/60] gitignore .clang-format --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0c997c8d..bce075d5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ features.h *.iml # CMake files -features.h \ No newline at end of file +features.h + +# Clang format +.clang-format \ No newline at end of file From 563e170e28dfcd99c594437b5e830d1a58f2489c Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 14:58:29 +0900 Subject: [PATCH 31/60] remove clang-format --- .clang-format | 8 -------- .gitignore | 3 --- 2 files changed, 11 deletions(-) delete mode 100644 .clang-format diff --git a/.clang-format b/.clang-format deleted file mode 100644 index 63a4b87f..00000000 --- a/.clang-format +++ /dev/null @@ -1,8 +0,0 @@ -BasedOnStyle: LLVM -IndentWidth: 2 -AlignConsecutiveAssignments: true -AlignConsecutiveMacros: true -AlignConsecutiveDeclarations: true -BreakBeforeBraces: Linux -AllowShortIfStatementsOnASingleLine: false -UseTab: ForIndentation \ No newline at end of file diff --git a/.gitignore b/.gitignore index bce075d5..9f253318 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,5 @@ features.h .idea *.iml -# CMake files -features.h - # Clang format .clang-format \ No newline at end of file From 1f3eac982e485db1812444104171a06a2ac34697 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 15:00:34 +0900 Subject: [PATCH 32/60] revert format --- program/c/src/oracle/oracle.h | 395 ++++++++++++++++++---------------- 1 file changed, 210 insertions(+), 185 deletions(-) diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index 4a32eff0..61fbc7b5 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -1,8 +1,8 @@ #pragma once -#include "features.h" -#include "util/compat_stdint.h" #include +#include "util/compat_stdint.h" +#include "features.h" #ifdef __cplusplus extern "C" { @@ -12,57 +12,61 @@ extern "C" { #define PC_MAGIC 0xa1b2c3d4 // current program version -#define PC_VERSION 2 +#define PC_VERSION 2 // max latency in slots between send and receive -#define PC_MAX_SEND_LATENCY 25 +#define PC_MAX_SEND_LATENCY 25 // various size constants -#define PC_PUBKEY_SIZE 32 -#define PC_PUBKEY_SIZE_64 (PC_PUBKEY_SIZE / sizeof(uint64_t)) -#define PC_MAP_TABLE_SIZE 640 +#define PC_PUBKEY_SIZE 32 +#define PC_PUBKEY_SIZE_64 (PC_PUBKEY_SIZE/sizeof(uint64_t)) +#define PC_MAP_TABLE_SIZE 640 -// Total price component slots available -#define PC_NUM_COMP_PYTHNET 127 + // Total price component slots available +#define PC_NUM_COMP_PYTHNET 127 // PC_NUM_COMP - number of price components in use -// Not whole PC_NUM_COMP_PYTHNET because of stack issues appearing in -// upd_aggregate() +// Not whole PC_NUM_COMP_PYTHNET because of stack issues appearing in upd_aggregate() #define PC_NUM_COMP 64 -#define PC_PROD_ACC_SIZE 512 -#define PC_EXP_DECAY -9 + +#define PC_PROD_ACC_SIZE 512 +#define PC_EXP_DECAY -9 #ifndef PC_HEAP_START #define PC_HEAP_START (0x300000000) #endif // price types -#define PC_PTYPE_UNKNOWN 0 -#define PC_PTYPE_PRICE 1 +#define PC_PTYPE_UNKNOWN 0 +#define PC_PTYPE_PRICE 1 // symbol status -#define PC_STATUS_UNKNOWN 0 -#define PC_STATUS_TRADING 1 -#define PC_STATUS_HALTED 2 -#define PC_STATUS_AUCTION 3 -#define PC_STATUS_IGNORED 4 +#define PC_STATUS_UNKNOWN 0 +#define PC_STATUS_TRADING 1 +#define PC_STATUS_HALTED 2 +#define PC_STATUS_AUCTION 3 +#define PC_STATUS_IGNORED 4 // account types -#define PC_ACCTYPE_MAPPING 1 -#define PC_ACCTYPE_PRODUCT 2 -#define PC_ACCTYPE_PRICE 3 -#define PC_ACCTYPE_TEST 4 -#define PC_ACCTYPE_PERMISSIONS 5 +#define PC_ACCTYPE_MAPPING 1 +#define PC_ACCTYPE_PRODUCT 2 +#define PC_ACCTYPE_PRICE 3 +#define PC_ACCTYPE_TEST 4 +#define PC_ACCTYPE_PERMISSIONS 5 + // Compute budget requested per price update instruction -// The biggest instruction appears to be about ~10300 CUs, so overestimate by -// 100%. +// The biggest instruction appears to be about ~10300 CUs, so overestimate by 100%. #define CU_BUDGET_PER_IX 20000 // binary version of sysvar_clock account id -const uint64_t sysvar_clock[] = {0xc974c71817d5a706UL, 0xb65e1d6998635628UL, - 0x5c6d4b9ba3b85e8bUL, 0x215b5573UL}; +const uint64_t sysvar_clock[] = { + 0xc974c71817d5a706UL, + 0xb65e1d6998635628UL, + 0x5c6d4b9ba3b85e8bUL, + 0x215b5573UL +}; // compute budget program id in hex (but wrong endianness) /* @@ -74,131 +78,137 @@ e79bc372baadecff // correct const uint64_t compute_budget_program_id[] = { - 0x321721e56f460603UL, 0xe79bc372baadecffUL, 0x6b12f7c5bbe58cbcUL, - 0x403a9b432cUL}; + 0x321721e56f460603UL, + 0xe79bc372baadecffUL, + 0x6b12f7c5bbe58cbcUL, + 0x403a9b432cUL +}; // public key of symbol or publisher account -typedef union pc_pub_key { +typedef union pc_pub_key +{ uint8_t k1_[PC_PUBKEY_SIZE]; uint64_t k8_[PC_PUBKEY_SIZE_64]; } pc_pub_key_t; -static_assert(sizeof(pc_pub_key_t) == 32, ""); +static_assert( sizeof( pc_pub_key_t ) == 32, "" ); // account header information -typedef struct pc_acc { - uint32_t magic_; // pyth magic number - uint32_t ver_; // program/account version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account +typedef struct pc_acc +{ + uint32_t magic_; // pyth magic number + uint32_t ver_; // program/account version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account } pc_acc_t; -static_assert(sizeof(pc_acc_t) == 16, ""); +static_assert( sizeof( pc_acc_t ) == 16, "" ); // hash table of symbol to price account mappings -typedef struct pc_map_table { - uint32_t magic_; // pyth magic number - uint32_t ver_; // program/account version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - uint32_t num_; // number of symbols - uint32_t unused_; // 64bit padding - pc_pub_key_t next_; // next mapping account in chain - pc_pub_key_t prod_[PC_MAP_TABLE_SIZE]; // product accounts +typedef struct pc_map_table +{ + uint32_t magic_; // pyth magic number + uint32_t ver_; // program/account version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + uint32_t num_; // number of symbols + uint32_t unused_; // 64bit padding + pc_pub_key_t next_; // next mapping account in chain + pc_pub_key_t prod_[PC_MAP_TABLE_SIZE]; // product accounts } pc_map_table_t; -static_assert(sizeof(pc_map_table_t) == 20536, ""); +static_assert( sizeof( pc_map_table_t ) == 20536, "" ); // variable length string -typedef struct pc_str { - uint8_t len_; - char data_[]; +typedef struct pc_str +{ + uint8_t len_; + char data_[]; } pc_str_t; -static_assert(sizeof(pc_str_t) == 1, ""); +static_assert( sizeof( pc_str_t ) == 1, "" ); // product reference data -typedef struct pc_prod { - uint32_t magic_; - uint32_t ver_; // program version - uint32_t type_; // account type - uint32_t size_; // size of populated region of account - pc_pub_key_t px_acc_; // first price (or quote) account +typedef struct pc_prod +{ + uint32_t magic_; + uint32_t ver_; // program version + uint32_t type_; // account type + uint32_t size_; // size of populated region of account + pc_pub_key_t px_acc_; // first price (or quote) account // variable number of reference key/value attribute pairs // stored as strings (pc_str_t) } pc_prod_t; -static_assert(sizeof(pc_prod_t) == 48, ""); +static_assert( sizeof( pc_prod_t ) == 48, "" ); // price et al. for some component or aggregate -typedef struct pc_price_info { - int64_t price_; // price per ptype_ - uint64_t conf_; // price confidence interval - uint32_t status_; // symbol status as of last update - uint32_t corp_act_status_; // corp action status as of last update - uint64_t pub_slot_; // publish slot of price +typedef struct pc_price_info +{ + int64_t price_; // price per ptype_ + uint64_t conf_; // price confidence interval + uint32_t status_; // symbol status as of last update + uint32_t corp_act_status_; // corp action status as of last update + uint64_t pub_slot_; // publish slot of price } pc_price_info_t; -static_assert(sizeof(pc_price_info_t) == 32, ""); +static_assert( sizeof( pc_price_info_t ) == 32, "" ); // published component price for contributing provider -typedef struct pc_price_comp { - pc_pub_key_t pub_; // publishing key of component price - pc_price_info_t agg_; // price used in aggregate calc - pc_price_info_t latest_; // latest contributed prices +typedef struct pc_price_comp +{ + pc_pub_key_t pub_; // publishing key of component price + pc_price_info_t agg_; // price used in aggregate calc + pc_price_info_t latest_; // latest contributed prices } pc_price_comp_t; -static_assert(sizeof(pc_price_comp_t) == 96, ""); +static_assert( sizeof( pc_price_comp_t ) == 96, "" ); // time-weighted exponential moving average -typedef struct pc_ema { - int64_t val_; // current value of ema - int64_t numer_; // numerator at full precision - int64_t denom_; // denominator at full precision +typedef struct pc_ema +{ + int64_t val_; // current value of ema + int64_t numer_; // numerator at full precision + int64_t denom_; // denominator at full precision } pc_ema_t; -static_assert(sizeof(pc_ema_t) == 24, ""); +static_assert( sizeof( pc_ema_t ) == 24, "" ); // price account containing aggregate and all component prices -typedef struct pc_price { - uint32_t magic_; // pyth magic number - uint32_t ver_; // program version - uint32_t type_; // account type - uint32_t size_; // price account size - uint32_t ptype_; // price or calculation type - int32_t expo_; // price exponent - uint32_t num_; // number of component prices - uint32_t num_qt_; // number of quoters that make up aggregate - uint64_t last_slot_; // slot of last valid aggregate price - uint64_t valid_slot_; // valid on-chain slot of agg. price - pc_ema_t twap_; // time-weighted average price - pc_ema_t twac_; // time-weighted average conf interval - int64_t timestamp_; // unix timestamp of aggregate price - uint8_t min_pub_; // min publishers for valid price - int8_t message_sent_; // flag to indicate if the current aggregate price has - // been sent as a message to the message buffer, 0 if - // not sent, 1 if sent - uint8_t max_latency_; // configurable max latency in slots between send and - // receive - int8_t drv3_; // space for future derived values - int32_t drv4_; // space for future derived values - pc_pub_key_t prod_; // product id/ref-account - pc_pub_key_t next_; // next price account in list - uint64_t prev_slot_; // valid slot of previous aggregate with TRADING status - int64_t prev_price_; // aggregate price of previous aggregate with TRADING - // status - uint64_t prev_conf_; // confidence interval of previous aggregate with - // TRADING status - int64_t prev_timestamp_; // unix timestamp of previous aggregate with - // TRADING status - pc_price_info_t agg_; // aggregate price information - pc_price_comp_t comp_[PC_NUM_COMP]; // component prices +typedef struct pc_price +{ + uint32_t magic_; // pyth magic number + uint32_t ver_; // program version + uint32_t type_; // account type + uint32_t size_; // price account size + uint32_t ptype_; // price or calculation type + int32_t expo_; // price exponent + uint32_t num_; // number of component prices + uint32_t num_qt_; // number of quoters that make up aggregate + uint64_t last_slot_; // slot of last valid aggregate price + uint64_t valid_slot_; // valid on-chain slot of agg. price + pc_ema_t twap_; // time-weighted average price + pc_ema_t twac_; // time-weighted average conf interval + int64_t timestamp_; // unix timestamp of aggregate price + uint8_t min_pub_; // min publishers for valid price + int8_t message_sent_; // flag to indicate if the current aggregate price has been sent as a message to the message buffer, 0 if not sent, 1 if sent + uint8_t max_latency_; // configurable max latency in slots between send and receive + int8_t drv3_; // space for future derived values + int32_t drv4_; // space for future derived values + pc_pub_key_t prod_; // product id/ref-account + pc_pub_key_t next_; // next price account in list + uint64_t prev_slot_; // valid slot of previous aggregate with TRADING status + int64_t prev_price_; // aggregate price of previous aggregate with TRADING status + uint64_t prev_conf_; // confidence interval of previous aggregate with TRADING status + int64_t prev_timestamp_; // unix timestamp of previous aggregate with TRADING status + pc_price_info_t agg_; // aggregate price information + pc_price_comp_t comp_[PC_NUM_COMP];// component prices } pc_price_t; + /* -The value 240 is derived from the fixed size of the pc_price_t struct excluding -the size of the comp_ array. Here is the breakdown of the sizes (in bytes) for -each field within the pc_price_t struct: +The value 240 is derived from the fixed size of the pc_price_t struct excluding the size of the comp_ array. +Here is the breakdown of the sizes (in bytes) for each field within the pc_price_t struct: - magic_ (uint32_t): 4 bytes - ver_ (uint32_t): 4 bytes @@ -218,8 +228,7 @@ each field within the pc_price_t struct: - max_latency_ (uint8_t): 1 byte - drv3_ (int8_t): 1 byte - drv4_ (int32_t): 4 bytes -- prod_ (pc_pub_key_t): 32 bytes (assuming pc_pub_key_t is a 32-byte array or -struct) +- prod_ (pc_pub_key_t): 32 bytes (assuming pc_pub_key_t is a 32-byte array or struct) - next_ (pc_pub_key_t): 32 bytes (similar to prod_) - prev_slot_ (uint64_t): 8 bytes - prev_price_ (int64_t): 8 bytes @@ -227,21 +236,21 @@ struct) - prev_timestamp_ (int64_t): 8 bytes - agg_ (pc_price_info_t): 32 bytes -Adding up all these sizes gives us a total of 240 bytes for the fixed part of -the pc_price_t struct. The size of the comp_ array is variable and depends on -PC_NUM_COMP, hence it is calculated separately and added to the base size of 240 -bytes. +Adding up all these sizes gives us a total of 240 bytes for the fixed part of the pc_price_t struct. +The size of the comp_ array is variable and depends on PC_NUM_COMP, hence it is calculated separately and added to the base size of 240 bytes. */ -#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET \ - (240 + PC_NUM_COMP * sizeof(pc_price_comp_t)) +#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (240 \ + + PC_NUM_COMP * sizeof(pc_price_comp_t) \ + ) -static_assert(sizeof(pc_price_t) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, ""); +static_assert( sizeof( pc_price_t ) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, "" ); #undef PC_EXPECTED_PRICE_T_SIZE_PYTHNET -// This constant needs to be an upper bound of the price account size, it is -// used within pythd for ztsd. It is set tighly to the current price account + -// 96 component prices + 48 bytes for cumulative sums -const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof(pc_price_comp_t) + 48; + +// This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd. +// It is set tighly to the current price account + 96 component prices + 48 bytes for cumulative sums +const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof( pc_price_comp_t) + 48; + // command enumeration typedef enum { @@ -312,9 +321,10 @@ typedef enum { // key[1] price account [signer writable] e_cmd_set_min_pub, - // publish component price, never returning an error even if the update - // failed key[0] funding account [signer writable] key[1] price - // account [writable] key[2] sysvar_clock account [readable] + // publish component price, never returning an error even if the update failed + // key[0] funding account [signer writable] + // key[1] price account [writable] + // key[2] sysvar_clock account [readable] e_cmd_upd_price_no_fail_on_error, // resizes a price account so that it fits the Time Machine @@ -336,115 +346,129 @@ typedef enum { e_cmd_del_product, } command_t; -typedef struct cmd_hdr { - uint32_t ver_; - int32_t cmd_; +typedef struct cmd_hdr +{ + uint32_t ver_; + int32_t cmd_; } cmd_hdr_t; -static_assert(sizeof(cmd_hdr_t) == 8, ""); +static_assert( sizeof( cmd_hdr_t ) == 8, "" ); -typedef struct cmd_add_product { - uint32_t ver_; - int32_t cmd_; +typedef struct cmd_add_product +{ + uint32_t ver_; + int32_t cmd_; } cmd_add_product_t; -static_assert(sizeof(cmd_add_product_t) == 8, ""); +static_assert( sizeof( cmd_add_product_t ) == 8, "" ); -typedef struct cmd_upd_product { - uint32_t ver_; - int32_t cmd_; +typedef struct cmd_upd_product +{ + uint32_t ver_; + int32_t cmd_; // set of key-value pairs } cmd_upd_product_t; -static_assert(sizeof(cmd_upd_product_t) == 8, ""); +static_assert( sizeof( cmd_upd_product_t ) == 8, "" ); -typedef struct cmd_add_price { - uint32_t ver_; - int32_t cmd_; - int32_t expo_; - uint32_t ptype_; +typedef struct cmd_add_price +{ + uint32_t ver_; + int32_t cmd_; + int32_t expo_; + uint32_t ptype_; } cmd_add_price_t; -static_assert(sizeof(cmd_add_price_t) == 16, ""); +static_assert( sizeof( cmd_add_price_t ) == 16, "" ); -typedef struct cmd_init_price { - uint32_t ver_; - int32_t cmd_; - int32_t expo_; - uint32_t ptype_; +typedef struct cmd_init_price +{ + uint32_t ver_; + int32_t cmd_; + int32_t expo_; + uint32_t ptype_; } cmd_init_price_t; -static_assert(sizeof(cmd_init_price_t) == 16, ""); +static_assert( sizeof( cmd_init_price_t ) == 16, "" ); -typedef struct cmd_set_min_pub { - uint32_t ver_; - int32_t cmd_; - uint8_t min_pub_; +typedef struct cmd_set_min_pub +{ + uint32_t ver_; + int32_t cmd_; + uint8_t min_pub_; } cmd_set_min_pub_t; -static_assert(sizeof(cmd_set_min_pub_t) == 12, ""); +static_assert( sizeof( cmd_set_min_pub_t ) == 12, "" ); -typedef struct cmd_add_publisher { +typedef struct cmd_add_publisher +{ uint32_t ver_; int32_t cmd_; pc_pub_key_t pub_; } cmd_add_publisher_t; -static_assert(sizeof(cmd_add_publisher_t) == 40, ""); +static_assert( sizeof( cmd_add_publisher_t ) == 40, "" ); -typedef struct cmd_del_publisher { +typedef struct cmd_del_publisher +{ uint32_t ver_; int32_t cmd_; pc_pub_key_t pub_; } cmd_del_publisher_t; -static_assert(sizeof(cmd_del_publisher_t) == 40, ""); +static_assert( sizeof( cmd_del_publisher_t ) == 40, "" ); -typedef struct cmd_upd_price { - uint32_t ver_; - int32_t cmd_; - uint32_t status_; - uint32_t unused_; - int64_t price_; - uint64_t conf_; - uint64_t pub_slot_; +typedef struct cmd_upd_price +{ + uint32_t ver_; + int32_t cmd_; + uint32_t status_; + uint32_t unused_; + int64_t price_; + uint64_t conf_; + uint64_t pub_slot_; } cmd_upd_price_t; -static_assert(sizeof(cmd_upd_price_t) == 40, ""); +static_assert( sizeof( cmd_upd_price_t ) == 40, "" ); // structure of clock sysvar account -typedef struct sysvar_clock { - uint64_t slot_; - int64_t epoch_start_timestamp_; - uint64_t epoch_; - uint64_t leader_schedule_epoch_; - int64_t unix_timestamp_; +typedef struct sysvar_clock +{ + uint64_t slot_; + int64_t epoch_start_timestamp_; + uint64_t epoch_; + uint64_t leader_schedule_epoch_; + int64_t unix_timestamp_; } sysvar_clock_t; -static_assert(sizeof(sysvar_clock_t) == 40, ""); +static_assert( sizeof( sysvar_clock_t ) == 40, "" ); // compare if two pub_keys (accounts) are the same -inline bool pc_pub_key_equal(pc_pub_key_t *p1, pc_pub_key_t *p2) +inline bool pc_pub_key_equal( pc_pub_key_t *p1, pc_pub_key_t *p2 ) { - return p1->k8_[0] == p2->k8_[0] && p1->k8_[1] == p2->k8_[1] && - p1->k8_[2] == p2->k8_[2] && p1->k8_[3] == p2->k8_[3]; + return p1->k8_[0] == p2->k8_[0] && + p1->k8_[1] == p2->k8_[1] && + p1->k8_[2] == p2->k8_[2] && + p1->k8_[3] == p2->k8_[3]; } // check for null (zero) public key -inline bool pc_pub_key_is_zero(pc_pub_key_t *p) +inline bool pc_pub_key_is_zero( pc_pub_key_t *p ) { - return p->k8_[0] == 0UL && p->k8_[1] == 0UL && p->k8_[2] == 0UL && + return p->k8_[0] == 0UL && + p->k8_[1] == 0UL && + p->k8_[2] == 0UL && p->k8_[3] == 0UL; } // set public key to zero -inline void pc_pub_key_set_zero(pc_pub_key_t *p) +inline void pc_pub_key_set_zero( pc_pub_key_t *p ) { p->k8_[0] = p->k8_[1] = p->k8_[2] = p->k8_[3] = 0UL; } // assign one pub_key from another -inline void pc_pub_key_assign(pc_pub_key_t *tgt, pc_pub_key_t *src) +inline void pc_pub_key_assign( pc_pub_key_t *tgt, pc_pub_key_t *src ) { tgt->k8_[0] = src->k8_[0]; tgt->k8_[1] = src->k8_[1]; @@ -452,6 +476,7 @@ inline void pc_pub_key_assign(pc_pub_key_t *tgt, pc_pub_key_t *src) tgt->k8_[3] = src->k8_[3]; } + #ifdef __cplusplus } #endif From 1523216ede05a177d5d7efd417a32df2dac190f2 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 15:05:15 +0900 Subject: [PATCH 33/60] revert format --- program/c/src/oracle/upd_aggregate.h | 179 +++++++++++++-------------- 1 file changed, 86 insertions(+), 93 deletions(-) diff --git a/program/c/src/oracle/upd_aggregate.h b/program/c/src/oracle/upd_aggregate.h index a63190d9..94179940 100644 --- a/program/c/src/oracle/upd_aggregate.h +++ b/program/c/src/oracle/upd_aggregate.h @@ -1,28 +1,29 @@ #pragma once -#include "model/price_model.c" /* FIXME: HACK TO DEAL WITH DOCKER LINKAGE ISSUES */ -#include "model/price_model.h" #include "oracle.h" +#include "model/price_model.h" +#include "model/price_model.c" /* FIXME: HACK TO DEAL WITH DOCKER LINKAGE ISSUES */ #include "pd.h" #ifdef __cplusplus extern "C" { #endif -typedef struct pc_qset { - pd_t iprice_[PC_NUM_COMP]; - pd_t uprice_[PC_NUM_COMP]; - pd_t lprice_[PC_NUM_COMP]; - pd_t weight_[PC_NUM_COMP]; - int64_t decay_[1 + PC_MAX_SEND_LATENCY]; - int64_t fact_[PC_FACTOR_SIZE]; +typedef struct pc_qset +{ + pd_t iprice_[PC_NUM_COMP]; + pd_t uprice_[PC_NUM_COMP]; + pd_t lprice_[PC_NUM_COMP]; + pd_t weight_[PC_NUM_COMP]; + int64_t decay_[1+PC_MAX_SEND_LATENCY]; + int64_t fact_[PC_FACTOR_SIZE]; } pc_qset_t; // initialize quote-set temporary data in heap area static pc_qset_t *qset_new() { // allocate off heap - pc_qset_t *qs = (pc_qset_t *)PC_HEAP_START; + pc_qset_t *qs = (pc_qset_t*)PC_HEAP_START; // sqrt of numbers 1 to 25 for decaying conf. interval based on slot delay qs->decay_[0] = 1000000000L; @@ -53,129 +54,124 @@ static pc_qset_t *qset_new() qs->decay_[25] = 5000000000L; // powers of 10 for use in decimal arithmetic scaling - qs->fact_[0] = 1L; - qs->fact_[1] = 10L; - qs->fact_[2] = 100L; - qs->fact_[3] = 1000L; - qs->fact_[4] = 10000L; - qs->fact_[5] = 100000L; - qs->fact_[6] = 1000000L; - qs->fact_[7] = 10000000L; - qs->fact_[8] = 100000000L; - qs->fact_[9] = 1000000000L; - qs->fact_[10] = 10000000000L; - qs->fact_[11] = 100000000000L; - qs->fact_[12] = 1000000000000L; - qs->fact_[13] = 10000000000000L; - qs->fact_[14] = 100000000000000L; - qs->fact_[15] = 1000000000000000L; - qs->fact_[16] = 10000000000000000L; - qs->fact_[17] = 100000000000000000L; + qs->fact_[0] = 1L; + qs->fact_[1] = 10L; + qs->fact_[2] = 100L; + qs->fact_[3] = 1000L; + qs->fact_[4] = 10000L; + qs->fact_[5] = 100000L; + qs->fact_[6] = 1000000L; + qs->fact_[7] = 10000000L; + qs->fact_[8] = 100000000L; + qs->fact_[9] = 1000000000L; + qs->fact_[10] = 10000000000L; + qs->fact_[11] = 100000000000L; + qs->fact_[12] = 1000000000000L; + qs->fact_[13] = 10000000000000L; + qs->fact_[14] = 100000000000000L; + qs->fact_[15] = 1000000000000000L; + qs->fact_[16] = 10000000000000000L; + qs->fact_[17] = 100000000000000000L; return qs; } -static void upd_ema(pc_ema_t *ptr, pd_t *val, pd_t *conf, int64_t nslot, - pc_qset_t *qs, int32_t expo) +static void upd_ema( + pc_ema_t *ptr, pd_t *val, pd_t *conf, int64_t nslot, pc_qset_t *qs, int32_t expo + ) { pd_t numer[1], denom[1], cwgt[1], wval[1], decay[1], diff[1], one[1]; - pd_new(one, 100000000L, -8); - if (conf->v_) { - pd_div(cwgt, one, conf); + pd_new( one, 100000000L, -8 ); + if ( conf->v_ ) { + pd_div( cwgt, one, conf ); } else { - pd_set(cwgt, one); + pd_set( cwgt, one ); } - if (nslot > PD_EMA_MAX_DIFF) { + if ( nslot > PD_EMA_MAX_DIFF ) { // initial condition - pd_mul(numer, val, cwgt); - pd_set(denom, cwgt); + pd_mul( numer, val, cwgt ); + pd_set( denom, cwgt ); } else { // compute decay factor - pd_new(diff, nslot, 0); - pd_new(decay, PD_EMA_DECAY, PD_EMA_EXPO); - pd_mul(decay, decay, diff); - pd_add(decay, decay, one, qs->fact_); + pd_new( diff, nslot, 0 ); + pd_new( decay, PD_EMA_DECAY, PD_EMA_EXPO ); + pd_mul( decay, decay, diff ); + pd_add( decay, decay, one, qs->fact_ ); // compute numer/denom and new value from decay factor - pd_load(numer, ptr->numer_); - pd_load(denom, ptr->denom_); - pd_mul(numer, numer, decay); - pd_mul(wval, val, cwgt); - pd_add(numer, numer, wval, qs->fact_); - pd_mul(denom, denom, decay); - pd_add(denom, denom, cwgt, qs->fact_); - pd_div(val, numer, denom); + pd_load( numer, ptr->numer_ ); + pd_load( denom, ptr->denom_ ); + pd_mul( numer, numer, decay ); + pd_mul( wval, val, cwgt ); + pd_add( numer, numer, wval, qs->fact_ ); + pd_mul( denom, denom, decay ); + pd_add( denom, denom, cwgt, qs->fact_ ); + pd_div( val, numer, denom ); } // adjust and store results - pd_adjust(val, expo, qs->fact_); - ptr->val_ = val->v_; + pd_adjust( val, expo, qs->fact_ ); + ptr->val_ = val->v_; int64_t numer1, denom1; - if (pd_store(&numer1, numer) && pd_store(&denom1, denom)) { + if ( pd_store( &numer1, numer ) && pd_store( &denom1, denom ) ) { ptr->numer_ = numer1; ptr->denom_ = denom1; } } -static inline void upd_twap(pc_price_t *ptr, int64_t nslots) +static inline void upd_twap( + pc_price_t *ptr, int64_t nslots ) { - pc_qset_t *qs = qset_new(); + pc_qset_t *qs = qset_new( ); pd_t px[1], conf[1]; - pd_new_scale(px, ptr->agg_.price_, ptr->expo_); - pd_new_scale(conf, (int64_t)(ptr->agg_.conf_), ptr->expo_); - upd_ema(&ptr->twap_, px, conf, nslots, qs, ptr->expo_); - upd_ema(&ptr->twac_, conf, conf, nslots, qs, ptr->expo_); + pd_new_scale( px, ptr->agg_.price_, ptr->expo_ ); + pd_new_scale( conf, ( int64_t )( ptr->agg_.conf_ ), ptr->expo_ ); + upd_ema( &ptr->twap_, px, conf, nslots, qs, ptr->expo_ ); + upd_ema( &ptr->twac_, conf, conf, nslots, qs, ptr->expo_ ); } // update aggregate price -static inline bool upd_aggregate(pc_price_t *ptr, uint64_t slot, - int64_t timestamp) +static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timestamp ) { // update aggregate details ready for next slot - ptr->valid_slot_ = ptr->agg_.pub_slot_; // valid slot-time of agg. price - ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price - ptr->timestamp_ = timestamp; + ptr->valid_slot_ = ptr->agg_.pub_slot_;// valid slot-time of agg. price + ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price + ptr->timestamp_ = timestamp; // identify valid quotes // compute the aggregate prices and ranges - int64_t agg_price; - int64_t agg_conf; + int64_t agg_price; + int64_t agg_conf; { uint32_t numv = 0; uint32_t nprcs = (uint32_t)0; - int64_t - prcs[PC_NUM_COMP * 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: - // DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) - for (uint32_t i = 0; i != ptr->num_; ++i) { + int64_t prcs[ PC_NUM_COMP * 3 ]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + for ( uint32_t i = 0; i != ptr->num_; ++i ) { pc_price_comp_t *iptr = &ptr->comp_[i]; // copy contributing price to aggregate snapshot iptr->agg_ = iptr->latest_; // add quote to sorted permutation array if it is valid - int64_t slot_diff = (int64_t)slot - (int64_t)(iptr->agg_.pub_slot_); + int64_t slot_diff = ( int64_t )slot - ( int64_t )( iptr->agg_.pub_slot_ ); int64_t price = iptr->agg_.price_; - int64_t conf = (int64_t)(iptr->agg_.conf_); - int64_t max_latency = - ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY; - if (iptr->agg_.status_ == PC_STATUS_TRADING && - // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < - // INT64_MAX These checks ensure that price - conf and price + conf do - // not overflow. - (int64_t)0 < conf && (INT64_MIN + conf) <= price && - price <= (INT64_MAX - conf) && - // slot_diff is implicitly >= 0 due to the check in Rust code ensuring - // publishing_slot is always less than or equal to the current slot. - slot_diff <= max_latency) { - numv += 1; - prcs[nprcs++] = price - conf; - prcs[nprcs++] = price; - prcs[nprcs++] = price + conf; + int64_t conf = ( int64_t )( iptr->agg_.conf_ ); + int64_t max_latency = ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY; + if ( iptr->agg_.status_ == PC_STATUS_TRADING && + // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < INT64_MAX + // These checks ensure that price - conf and price + conf do not overflow. + (int64_t)0 < conf && (INT64_MIN + conf) <= price && price <= (INT64_MAX-conf) && + // slot_diff is implicitly >= 0 due to the check in Rust code ensuring publishing_slot is always less than or equal to the current slot. + slot_diff <= max_latency ) { + numv += 1; + prcs[ nprcs++ ] = price - conf; + prcs[ nprcs++ ] = price; + prcs[ nprcs++ ] = price + conf; } } // too few valid quotes ptr->num_qt_ = numv; - if (numv == 0 || numv < ptr->min_pub_) { + if ( numv == 0 || numv < ptr->min_pub_ ) { ptr->agg_.status_ = PC_STATUS_UNKNOWN; return false; } @@ -184,11 +180,8 @@ static inline bool upd_aggregate(pc_price_t *ptr, uint64_t slot, // note: numv>0 and nprcs = 3*numv at this point int64_t agg_p25; int64_t agg_p75; - int64_t scratch[PC_NUM_COMP * - 3]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK - // THIS FITS INTO STACK FRAME LIMIT) - price_model_core((uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, - scratch); + int64_t scratch[ PC_NUM_COMP * 3 ]; // ~0.75KiB for current PC_NUM_COMP (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + price_model_core( (uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, scratch ); // get the left and right confidences // note that as valid quotes have positive prices currently and @@ -203,7 +196,7 @@ static inline bool upd_aggregate(pc_price_t *ptr, uint64_t slot, // if the confidences end up at zero, we abort // this is paranoia as it is currently not possible when nprcs>2 and // positive confidences given the current pricing model - if (agg_conf <= (int64_t)0) { + if( agg_conf <= (int64_t)0 ) { ptr->agg_.status_ = PC_STATUS_UNKNOWN; return false; } From 4596c6ebd14aeabaa0696d1ae33af55cb243b956 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 15:06:48 +0900 Subject: [PATCH 34/60] format --- program/rust/src/processor/upd_price.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index ff49f03f..e09e9c8f 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -230,13 +230,11 @@ pub fn upd_price( // will send the message. price_data.message_sent_ = 0; } - // Encapsulate TWAP update logic in a function to minimize unsafe block scope. unsafe { c_upd_twap(price_account.try_borrow_mut_data()?.as_mut_ptr(), agg_diff); } } - // Reload price data as a struct after c_upd_aggregate() borrow is dropped let mut price_data = load_checked::(price_account, cmd_args.header.version)?; From 11d6086315fbe7cc7bf7f64114ff77213748304e Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 15:07:19 +0900 Subject: [PATCH 35/60] revert format --- program/rust/src/tests/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index 59ae7e5b..8208d153 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -26,5 +26,6 @@ mod test_upd_price_no_fail_on_error; mod test_upd_product; mod test_utils; + mod test_twap; mod test_upd_price_v2; From ac7508d81696554322d30219a1de310324510ecd Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 28 Mar 2024 15:08:25 +0900 Subject: [PATCH 36/60] revert format --- scripts/build-bpf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-bpf.sh b/scripts/build-bpf.sh index 9460391d..fd85e4fa 100755 --- a/scripts/build-bpf.sh +++ b/scripts/build-bpf.sh @@ -27,4 +27,4 @@ sha256sum ./target/**/*.so echo "Checking size of pyth_oracle.so for pythnet" ./scripts/check-size.sh 88429 mkdir -p target/pyth/pythnet/ -mv target/deploy/pyth_oracle.so target/pyth/pythnet/pyth_oracle_pythnet.so \ No newline at end of file +mv target/deploy/pyth_oracle.so target/pyth/pythnet/pyth_oracle_pythnet.so From 77cced8b728d87074e699ff4f60265e810131ce5 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 2 Apr 2024 12:25:17 +0900 Subject: [PATCH 37/60] fix comment --- program/rust/src/tests/test_ema.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index 4f22370d..c682c470 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -86,7 +86,7 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box Result<(), Box Date: Tue, 2 Apr 2024 12:36:01 +0900 Subject: [PATCH 38/60] remove comment --- program/rust/src/tests/test_upd_price.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 55290d85..5ac12549 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -248,7 +248,7 @@ fn test_upd_price() { assert_eq!(price_data.valid_slot_, 5); assert_eq!(price_data.agg_.pub_slot_, 6); assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); // in pythnet aggregation is on the same slot so status turns to unknown + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); } // Crank one more time and aggregate should be unknown From caaecc169f725a36aa91e804e0362ae23a0f083b Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 2 Apr 2024 13:00:01 +0900 Subject: [PATCH 39/60] add comment --- program/rust/src/processor/upd_price.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index e09e9c8f..eef7c089 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -150,6 +150,9 @@ pub fn upd_price( OracleError::PermissionViolation.into(), )?; + // We assign the current aggregate price to latest_aggregate_price before calling c_upd_aggregate because + // c_upd_aggregate directly modifies the memory of price_data.agg_ to update its values. This is crucial for + // comparisons or operations that rely on the aggregate price state before the update such as slots_since_last_update. latest_aggregate_price = price_data.agg_; let latest_publisher_price = price_data.comp_[publisher_index].latest_; @@ -162,7 +165,6 @@ pub fn upd_price( )?; } - // Try to update the publisher's price if is_component_update(cmd_args)? { // IMPORTANT: If the publisher does not meet the price/conf @@ -194,7 +196,6 @@ pub fn upd_price( price_data.prev_timestamp_ = clock.unix_timestamp; }; - let updated = unsafe { // NOTE: c_upd_aggregate must use a raw pointer to price // data. Solana's `.borrow_*` methods require exclusive From 5121d1d8353ec11996cd3ad026c35f64f3c1b7fe Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 2 Apr 2024 13:13:51 +0900 Subject: [PATCH 40/60] refactor --- program/rust/src/processor/upd_price.rs | 42 ++++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index eef7c089..ba4d680a 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -165,36 +165,34 @@ pub fn upd_price( )?; } - // Try to update the publisher's price - if is_component_update(cmd_args)? { - // IMPORTANT: If the publisher does not meet the price/conf - // ratio condition, its price will not count for the next - // aggregate. - let status: u32 = - get_status_for_conf_price_ratio(cmd_args.price, cmd_args.confidence, cmd_args.status)?; + // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. + let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; - { - let mut price_data = - load_checked::(price_account, cmd_args.header.version)?; + // Extend the scope of the mutable borrow of price_data + { + let mut price_data = load_checked::(price_account, cmd_args.header.version)?; + + // Update the publisher's price + if is_component_update(cmd_args)? { + let status: u32 = get_status_for_conf_price_ratio( + cmd_args.price, + cmd_args.confidence, + cmd_args.status, + )?; let publisher_price = &mut price_data.comp_[publisher_index].latest_; publisher_price.price_ = cmd_args.price; publisher_price.conf_ = cmd_args.confidence; publisher_price.status_ = status; publisher_price.pub_slot_ = cmd_args.publishing_slot; } - } - - // get number of slots from last update - let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; - // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. - if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { - let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - price_data.prev_slot_ = price_data.agg_.pub_slot_; - price_data.prev_price_ = price_data.agg_.price_; - price_data.prev_conf_ = price_data.agg_.conf_; - price_data.prev_timestamp_ = clock.unix_timestamp; - }; + if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { + price_data.prev_slot_ = price_data.agg_.pub_slot_; + price_data.prev_price_ = price_data.agg_.price_; + price_data.prev_conf_ = price_data.agg_.conf_; + price_data.prev_timestamp_ = clock.unix_timestamp; + } + } let updated = unsafe { // NOTE: c_upd_aggregate must use a raw pointer to price From b8f34844a3b6c751ba1b457d399f188b483b708c Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 3 Apr 2024 13:31:44 +0900 Subject: [PATCH 41/60] add guard for first price update after deployment --- program/rust/src/processor/upd_price.rs | 28 ++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index ba4d680a..15117e8d 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -165,7 +165,6 @@ pub fn upd_price( )?; } - // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; // Extend the scope of the mutable borrow of price_data @@ -186,6 +185,7 @@ pub fn upd_price( publisher_price.pub_slot_ = cmd_args.publishing_slot; } + // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { price_data.prev_slot_ = price_data.agg_.pub_slot_; price_data.prev_price_ = price_data.agg_.price_; @@ -213,8 +213,30 @@ pub fn upd_price( { let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - // check if its the first price update in the slot - if slots_since_last_update > 0 { + + // Check if this is the first update by checking if previous values are uninitialized. + // PriceEma implements Zeroable trait, so they are initialized to 0 and we check for that to determine if they are uninitialized. + let is_prev_twap_uninitialized = price_data.prev_twap_.val_ == 0 + && price_data.prev_twap_.numer_ == 0 + && price_data.prev_twap_.denom_ == 0; + let is_prev_twac_uninitialized = price_data.prev_twac_.val_ == 0 + && price_data.prev_twac_.numer_ == 0 + && price_data.prev_twac_.denom_ == 0; + let is_prev_price_cumulative_uninitialized = price_data.prev_price_cumulative.price + == 0 + && price_data.prev_price_cumulative.conf == 0 + && price_data.prev_price_cumulative.num_down_slots == 0 + && price_data.prev_price_cumulative.unused == 0; + let is_prev_ema_and_twap_uninitialized = is_prev_twap_uninitialized + && is_prev_twac_uninitialized + && is_prev_price_cumulative_uninitialized; + + // Multiple price updates may occur within the same slot. Updates within the same slot will + // use the previously calculated values (prev_twap, prev_twac, and prev_price_cumulative) + // from the last successful aggregated price update as their basis for recalculation. This + // ensures that each update within a slot builds upon the last and not the twap/twac/price_cumulative + // that is calculated right after the publishers' individual price updates. + if slots_since_last_update > 0 || is_prev_ema_and_twap_uninitialized { price_data.prev_twap_ = price_data.twap_; price_data.prev_twac_ = price_data.twac_; price_data.prev_price_cumulative = price_data.price_cumulative; From f2c96f009df514c7906123df22dd62453bd63554 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 3 Apr 2024 13:43:46 +0900 Subject: [PATCH 42/60] add back deleted test in test_publish --- program/rust/src/tests/test_publish.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/program/rust/src/tests/test_publish.rs b/program/rust/src/tests/test_publish.rs index 26eb463b..f10489d8 100644 --- a/program/rust/src/tests/test_publish.rs +++ b/program/rust/src/tests/test_publish.rs @@ -64,6 +64,20 @@ async fn test_publish() { .await .unwrap(); + { + let price_data = sim + .get_account_data_as::(price) + .await + .unwrap(); + + assert_eq!(price_data.comp_[0].latest_.price_, 150); + assert_eq!(price_data.comp_[0].latest_.conf_, 7); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.comp_[0].agg_.price_, 150); + assert_eq!(price_data.comp_[0].agg_.conf_, 7); + assert_eq!(price_data.comp_[0].agg_.status_, PC_STATUS_TRADING); + } + sim.warp_to_slot(2).await.unwrap(); sim.upd_price( &publisher, From 6bf147c4d148f1d5f11716e4ffd54f47e027e713 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 3 Apr 2024 13:58:55 +0900 Subject: [PATCH 43/60] add tests for prev values to test_upd_price --- program/rust/src/tests/test_upd_price.rs | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 5ac12549..5e2b2183 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -80,6 +80,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 1); assert_eq!(price_data.agg_.price_, 42); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 0); + assert_eq!(price_data.prev_price_, 0); + assert_eq!(price_data.prev_conf_, 0); + assert_eq!(price_data.prev_timestamp_, 0); } // add some prices for current slot - get rejected @@ -108,6 +112,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 1); assert_eq!(price_data.agg_.price_, 42); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 0); + assert_eq!(price_data.prev_price_, 0); + assert_eq!(price_data.prev_conf_, 0); + assert_eq!(price_data.prev_timestamp_, 0); } // add next price in new slot triggering snapshot and aggregate calc @@ -135,6 +143,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 3); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 1); + assert_eq!(price_data.prev_price_, 42); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } // next price doesn't change but slot does @@ -161,6 +173,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 4); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 3); + assert_eq!(price_data.prev_price_, 81); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } // next price doesn't change and neither does aggregate but slot does @@ -187,6 +203,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 5); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 4); + assert_eq!(price_data.prev_price_, 81); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } // try to publish back-in-time @@ -215,6 +235,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 5); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 4); + assert_eq!(price_data.prev_price_, 81); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } populate_instruction(&mut instruction_data, 50, 20, 5); @@ -249,6 +273,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 6); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + assert_eq!(price_data.prev_slot_, 5); + assert_eq!(price_data.prev_price_, 81); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } // Crank one more time and aggregate should be unknown @@ -276,6 +304,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 7); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + assert_eq!(price_data.prev_slot_, 5); + assert_eq!(price_data.prev_price_, 81); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } // Negative prices are accepted @@ -302,6 +334,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 8); assert_eq!(price_data.agg_.price_, -100); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 5); + assert_eq!(price_data.prev_price_, 81); + assert_eq!(price_data.prev_conf_, 2); + assert_eq!(price_data.prev_timestamp_, 0); } // Crank again for aggregate @@ -329,6 +365,10 @@ fn test_upd_price() { assert_eq!(price_data.agg_.pub_slot_, 9); assert_eq!(price_data.agg_.price_, -100); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 8); + assert_eq!(price_data.prev_price_, -100); + assert_eq!(price_data.prev_conf_, 1); + assert_eq!(price_data.prev_timestamp_, 0); } } From 7e53f79000dc20708542fc51782b6fd9197c6dab Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 3 Apr 2024 14:04:46 +0900 Subject: [PATCH 44/60] update comment --- program/rust/src/tests/test_upd_price.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 5e2b2183..262b98da 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -118,7 +118,7 @@ fn test_upd_price() { assert_eq!(price_data.prev_timestamp_, 0); } - // add next price in new slot triggering snapshot and aggregate calc + // update new price in new slot, aggregate should be updated and prev values should be updated populate_instruction(&mut instruction_data, 81, 2, 2); update_clock_slot(&mut clock_account, 3); From 149dec4ef4df4fb6e2a19a782cc9bdb0d5910dba Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 3 Apr 2024 14:33:09 +0900 Subject: [PATCH 45/60] add test --- program/rust/src/tests/test_upd_aggregate.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/program/rust/src/tests/test_upd_aggregate.rs b/program/rust/src/tests/test_upd_aggregate.rs index 2747e1c7..7bb297c3 100644 --- a/program/rust/src/tests/test_upd_aggregate.rs +++ b/program/rust/src/tests/test_upd_aggregate.rs @@ -93,6 +93,19 @@ fn test_upd_aggregate() { )); } + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + + assert_eq!(price_data.agg_.price_, 100); + assert_eq!(price_data.agg_.conf_, 10); + assert_eq!(price_data.num_qt_, 1); + assert_eq!(price_data.timestamp_, 1); + assert_eq!(price_data.prev_slot_, 0); + assert_eq!(price_data.prev_price_, 0); + assert_eq!(price_data.prev_conf_, 0); + assert_eq!(price_data.prev_timestamp_, 0); + } + // single publisher { let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); From 0ff10615cda089046dfae72a81388d88a08894cd Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 3 Apr 2024 23:04:34 +0900 Subject: [PATCH 46/60] use last_slot_ instead of agg_.pub_slot_ --- program/rust/src/processor/upd_price.rs | 53 ++++++++++------------- program/rust/src/tests/test_ema.rs | 56 +++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 15117e8d..5edf7a8e 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -130,6 +130,7 @@ pub fn upd_price( let mut publisher_index: usize = 0; let latest_aggregate_price: PriceInfo; + let slots_since_last_update: u64; // The price_data borrow happens in a scope because it must be // dropped before we borrow again as raw data pointer for the C @@ -150,6 +151,9 @@ pub fn upd_price( OracleError::PermissionViolation.into(), )?; + // We use last_slot_ to calculate slots_since_last_update. This is because last_slot_ is updated after the aggregate price is updated successfully. + slots_since_last_update = clock.slot - price_data.last_slot_; + // We assign the current aggregate price to latest_aggregate_price before calling c_upd_aggregate because // c_upd_aggregate directly modifies the memory of price_data.agg_ to update its values. This is crucial for // comparisons or operations that rely on the aggregate price state before the update such as slots_since_last_update. @@ -165,8 +169,6 @@ pub fn upd_price( )?; } - let slots_since_last_update = clock.slot - latest_aggregate_price.pub_slot_; - // Extend the scope of the mutable borrow of price_data { let mut price_data = load_checked::(price_account, cmd_args.header.version)?; @@ -214,36 +216,25 @@ pub fn upd_price( let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - // Check if this is the first update by checking if previous values are uninitialized. - // PriceEma implements Zeroable trait, so they are initialized to 0 and we check for that to determine if they are uninitialized. - let is_prev_twap_uninitialized = price_data.prev_twap_.val_ == 0 - && price_data.prev_twap_.numer_ == 0 - && price_data.prev_twap_.denom_ == 0; - let is_prev_twac_uninitialized = price_data.prev_twac_.val_ == 0 - && price_data.prev_twac_.numer_ == 0 - && price_data.prev_twac_.denom_ == 0; - let is_prev_price_cumulative_uninitialized = price_data.prev_price_cumulative.price - == 0 - && price_data.prev_price_cumulative.conf == 0 - && price_data.prev_price_cumulative.num_down_slots == 0 - && price_data.prev_price_cumulative.unused == 0; - let is_prev_ema_and_twap_uninitialized = is_prev_twap_uninitialized - && is_prev_twac_uninitialized - && is_prev_price_cumulative_uninitialized; - - // Multiple price updates may occur within the same slot. Updates within the same slot will - // use the previously calculated values (prev_twap, prev_twac, and prev_price_cumulative) - // from the last successful aggregated price update as their basis for recalculation. This - // ensures that each update within a slot builds upon the last and not the twap/twac/price_cumulative - // that is calculated right after the publishers' individual price updates. - if slots_since_last_update > 0 || is_prev_ema_and_twap_uninitialized { - price_data.prev_twap_ = price_data.twap_; - price_data.prev_twac_ = price_data.twac_; - price_data.prev_price_cumulative = price_data.price_cumulative; + // Check if the program upgrade has happened in the current slot and aggregate price has been updated, if so, use the old logic to update twap/twac/price_cumulative. + // This is to ensure that twap/twac/price_cumulative are calculated correctly during the migration. + // We check if prev_twap_.denom_ is == 0 because when the program upgrade has happened, denom_ is initialized to 0 and it can only stay the same or increase while numer_ can be negative if prices are negative. + // And we check if slots_since_last_update == 0 to check if the aggregate price has been updated in the current slot. + if !(price_data.prev_twap_.denom_ == 0 && slots_since_last_update == 0) { + // Multiple price updates may occur within the same slot. Updates within the same slot will + // use the previously calculated values (prev_twap, prev_twac, and prev_price_cumulative) + // from the last successful aggregated price update as their basis for recalculation. This + // ensures that each update within a slot builds upon the last and not the twap/twac/price_cumulative + // that is calculated right after the publishers' individual price updates. + if slots_since_last_update > 0 { + price_data.prev_twap_ = price_data.twap_; + price_data.prev_twac_ = price_data.twac_; + price_data.prev_price_cumulative = price_data.price_cumulative; + } + price_data.twap_ = price_data.prev_twap_; + price_data.twac_ = price_data.prev_twac_; + price_data.price_cumulative = price_data.prev_price_cumulative; } - price_data.twap_ = price_data.prev_twap_; - price_data.twac_ = price_data.prev_twac_; - price_data.price_cumulative = price_data.prev_price_cumulative; price_data.update_price_cumulative()?; // We want to send a message every time the aggregate price updates. However, during the migration, // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index c682c470..d671cc91 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -9,6 +9,7 @@ use { }, c_oracle_header::{ PC_STATUS_TRADING, + PC_STATUS_UNKNOWN, PC_VERSION, }, deserialize::{ @@ -127,6 +128,61 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box(&price_account, PC_VERSION).unwrap(); + price_data.min_pub_ = 2; + price_data.num_ = 1; + } + + update_clock_slot(&mut clock_account, 3); + + populate_instruction(&mut instruction_data, 20, 1, 3); + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + assert_eq!(price_data.prev_twap_.val_, 10); + assert_eq!(price_data.prev_twac_.val_, 1); + assert_eq!(price_data.twap_.val_, 12); + assert_eq!(price_data.twac_.val_, 1); + } + + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.min_pub_ = 2; + price_data.num_ = 2; + } + + populate_instruction(&mut instruction_data, 40, 1, 3); + process_instruction( + &program_id, + &[ + funding_account_two.clone(), + price_account.clone(), + clock_account.clone(), + ], + &instruction_data, + )?; + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_twap_.val_, 12); + assert_eq!(price_data.prev_twac_.val_, 1); + assert_eq!(price_data.twap_.val_, 13); + assert_eq!(price_data.twac_.val_, 2); + } Ok(()) } From 6000143240cc87a6836277fa53e73bf44c2bc9a4 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 4 Apr 2024 11:18:22 +0900 Subject: [PATCH 47/60] add more asserts --- program/rust/src/tests/test_ema.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index d671cc91..0206c3d3 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -110,6 +110,14 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.prev_twap_.val_, 10); + assert_eq!(price_data.prev_twac_.val_, 1); + assert_eq!(price_data.twap_.val_, 15); + assert_eq!(price_data.twac_.val_, 1); + } + populate_instruction(&mut instruction_data, 30, 1, 2); process_instruction( &program_id, From eeccb2a3e5b909d55b3a8cfdeba93aad89d9edb7 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 4 Apr 2024 11:19:30 +0900 Subject: [PATCH 48/60] remove crank again asserts --- program/rust/src/tests/test_upd_price.rs | 62 ------------------------ 1 file changed, 62 deletions(-) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 262b98da..f0f59a2f 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -279,37 +279,6 @@ fn test_upd_price() { assert_eq!(price_data.prev_timestamp_, 0); } - // Crank one more time and aggregate should be unknown - populate_instruction(&mut instruction_data, 50, 20, 6); - update_clock_slot(&mut clock_account, 7); - - assert!(process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone() - ], - &instruction_data - ) - .is_ok()); - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 50); - assert_eq!(price_data.comp_[0].latest_.conf_, 20); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 6); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); - assert_eq!(price_data.valid_slot_, 6); - assert_eq!(price_data.agg_.pub_slot_, 7); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - assert_eq!(price_data.prev_slot_, 5); - assert_eq!(price_data.prev_price_, 81); - assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); - } - // Negative prices are accepted populate_instruction(&mut instruction_data, -100, 1, 7); update_clock_slot(&mut clock_account, 8); @@ -339,37 +308,6 @@ fn test_upd_price() { assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 0); } - - // Crank again for aggregate - populate_instruction(&mut instruction_data, -100, 1, 8); - update_clock_slot(&mut clock_account, 9); - - assert!(process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone() - ], - &instruction_data - ) - .is_ok()); - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, -100); - assert_eq!(price_data.comp_[0].latest_.conf_, 1); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 8); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 8); - assert_eq!(price_data.agg_.pub_slot_, 9); - assert_eq!(price_data.agg_.price_, -100); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.prev_slot_, 8); - assert_eq!(price_data.prev_price_, -100); - assert_eq!(price_data.prev_conf_, 1); - assert_eq!(price_data.prev_timestamp_, 0); - } } // Create an upd_price instruction with the provided parameters From 87ddfde5709d7c1a379fd20291a57b345e4d79b9 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 4 Apr 2024 23:24:25 +0900 Subject: [PATCH 49/60] fix --- program/rust/src/processor/upd_price.rs | 34 +++++++++++++++--------- program/rust/src/tests/test_upd_price.rs | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 5edf7a8e..cd9e9e07 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -131,6 +131,7 @@ pub fn upd_price( let mut publisher_index: usize = 0; let latest_aggregate_price: PriceInfo; let slots_since_last_update: u64; + let noninitial_price_update_after_program_upgrade: bool; // The price_data borrow happens in a scope because it must be // dropped before we borrow again as raw data pointer for the C @@ -154,6 +155,13 @@ pub fn upd_price( // We use last_slot_ to calculate slots_since_last_update. This is because last_slot_ is updated after the aggregate price is updated successfully. slots_since_last_update = clock.slot - price_data.last_slot_; + // Check if the program upgrade has happened in the current slot and aggregate price has been updated, if so, use the old logic to update twap/twac/price_cumulative. + // This is to ensure that twap/twac/price_cumulative are calculated correctly during the migration. + // We check if prev_twap_.denom_ is == 0 because when the program upgrade has happened, denom_ is initialized to 0 and it can only stay the same or increase while numer_ can be negative if prices are negative. + // And we check if slots_since_last_update == 0 to check if the aggregate price has been updated in the current slot. + noninitial_price_update_after_program_upgrade = + price_data.prev_twap_.denom_ == 0 && slots_since_last_update == 0; + // We assign the current aggregate price to latest_aggregate_price before calling c_upd_aggregate because // c_upd_aggregate directly modifies the memory of price_data.agg_ to update its values. This is crucial for // comparisons or operations that rely on the aggregate price state before the update such as slots_since_last_update. @@ -197,14 +205,18 @@ pub fn upd_price( } let updated = unsafe { - // NOTE: c_upd_aggregate must use a raw pointer to price - // data. Solana's `.borrow_*` methods require exclusive - // access, i.e. no other borrow can exist for the account. - c_upd_aggregate( - price_account.try_borrow_mut_data()?.as_mut_ptr(), - clock.slot, - clock.unix_timestamp, - ) + if noninitial_price_update_after_program_upgrade { + false + } else { + // NOTE: c_upd_aggregate must use a raw pointer to price + // data. Solana's `.borrow_*` methods require exclusive + // access, i.e. no other borrow can exist for the account. + c_upd_aggregate( + price_account.try_borrow_mut_data()?.as_mut_ptr(), + clock.slot, + clock.unix_timestamp, + ) + } }; // If the aggregate was successfully updated, calculate the difference and update TWAP. @@ -216,11 +228,7 @@ pub fn upd_price( let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - // Check if the program upgrade has happened in the current slot and aggregate price has been updated, if so, use the old logic to update twap/twac/price_cumulative. - // This is to ensure that twap/twac/price_cumulative are calculated correctly during the migration. - // We check if prev_twap_.denom_ is == 0 because when the program upgrade has happened, denom_ is initialized to 0 and it can only stay the same or increase while numer_ can be negative if prices are negative. - // And we check if slots_since_last_update == 0 to check if the aggregate price has been updated in the current slot. - if !(price_data.prev_twap_.denom_ == 0 && slots_since_last_update == 0) { + if !(noninitial_price_update_after_program_upgrade) { // Multiple price updates may occur within the same slot. Updates within the same slot will // use the previously calculated values (prev_twap, prev_twac, and prev_price_cumulative) // from the last successful aggregated price update as their basis for recalculation. This diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index f0f59a2f..3481f5de 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -299,7 +299,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.conf_, 1); assert_eq!(price_data.comp_[0].latest_.pub_slot_, 7); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 7); + assert_eq!(price_data.valid_slot_, 6); assert_eq!(price_data.agg_.pub_slot_, 8); assert_eq!(price_data.agg_.price_, -100); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); From c976989879430313ff645dbe8cd33c2c0180a73b Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 8 Apr 2024 12:59:17 +0900 Subject: [PATCH 50/60] address comments --- program/rust/src/processor/upd_price.rs | 35 ++++++++++++------------ program/rust/src/tests/test_upd_price.rs | 19 ++++++++----- program/rust/src/tests/test_utils.rs | 6 ++++ 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index cd9e9e07..99cbbcff 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -200,7 +200,7 @@ pub fn upd_price( price_data.prev_slot_ = price_data.agg_.pub_slot_; price_data.prev_price_ = price_data.agg_.price_; price_data.prev_conf_ = price_data.agg_.conf_; - price_data.prev_timestamp_ = clock.unix_timestamp; + price_data.prev_timestamp_ = price_data.timestamp_; } } @@ -221,28 +221,24 @@ pub fn upd_price( // If the aggregate was successfully updated, calculate the difference and update TWAP. if updated { - let agg_diff = (clock.slot as i64) - - load_checked::(price_account, cmd_args.header.version)?.prev_slot_ - as i64; { let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - if !(noninitial_price_update_after_program_upgrade) { - // Multiple price updates may occur within the same slot. Updates within the same slot will - // use the previously calculated values (prev_twap, prev_twac, and prev_price_cumulative) - // from the last successful aggregated price update as their basis for recalculation. This - // ensures that each update within a slot builds upon the last and not the twap/twac/price_cumulative - // that is calculated right after the publishers' individual price updates. - if slots_since_last_update > 0 { - price_data.prev_twap_ = price_data.twap_; - price_data.prev_twac_ = price_data.twac_; - price_data.prev_price_cumulative = price_data.price_cumulative; - } - price_data.twap_ = price_data.prev_twap_; - price_data.twac_ = price_data.prev_twac_; - price_data.price_cumulative = price_data.prev_price_cumulative; + // Multiple price updates may occur within the same slot. Updates within the same slot will + // use the previously calculated values (prev_twap, prev_twac, and prev_price_cumulative) + // from the last successful aggregated price update as their basis for recalculation. This + // ensures that each update within a slot builds upon the last and not the twap/twac/price_cumulative + // that is calculated right after the publishers' individual price updates. + if slots_since_last_update > 0 { + price_data.prev_twap_ = price_data.twap_; + price_data.prev_twac_ = price_data.twac_; + price_data.prev_price_cumulative = price_data.price_cumulative; } + price_data.twap_ = price_data.prev_twap_; + price_data.twac_ = price_data.prev_twac_; + price_data.price_cumulative = price_data.prev_price_cumulative; + price_data.update_price_cumulative()?; // We want to send a message every time the aggregate price updates. However, during the migration, // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag @@ -250,6 +246,9 @@ pub fn upd_price( // will send the message. price_data.message_sent_ = 0; } + let agg_diff = (clock.slot as i64) + - load_checked::(price_account, cmd_args.header.version)?.prev_slot_ + as i64; unsafe { c_upd_twap(price_account.try_borrow_mut_data()?.as_mut_ptr(), agg_diff); } diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 3481f5de..1870db37 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -21,6 +21,7 @@ use { processor::process_instruction, tests::test_utils::{ update_clock_slot, + update_clock_timestamp, AccountSetup, }, }, @@ -58,6 +59,7 @@ fn test_upd_price() { clock_account.is_writable = false; update_clock_slot(&mut clock_account, 1); + update_clock_timestamp(&mut clock_account, 1); assert!(process_instruction( &program_id, @@ -146,12 +148,14 @@ fn test_upd_price() { assert_eq!(price_data.prev_slot_, 1); assert_eq!(price_data.prev_price_, 42); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.prev_timestamp_, 1); } - // next price doesn't change but slot does + println!("test"); + // next price doesn't change but slot and timestamp does populate_instruction(&mut instruction_data, 81, 2, 3); update_clock_slot(&mut clock_account, 4); + update_clock_timestamp(&mut clock_account, 4); assert!(process_instruction( &program_id, &[ @@ -176,7 +180,8 @@ fn test_upd_price() { assert_eq!(price_data.prev_slot_, 3); assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.timestamp_, 4); // We only check for timestamp_ here because test_upd_price doesn't directly update timestamp_, this is updated through c_upd_aggregate which is tested in test_upd_aggregate, but we assert here to show that in subsequent asserts for prev_tim + assert_eq!(price_data.prev_timestamp_, 1); } // next price doesn't change and neither does aggregate but slot does @@ -206,7 +211,7 @@ fn test_upd_price() { assert_eq!(price_data.prev_slot_, 4); assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.prev_timestamp_, 4); } // try to publish back-in-time @@ -238,7 +243,7 @@ fn test_upd_price() { assert_eq!(price_data.prev_slot_, 4); assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.prev_timestamp_, 4); } populate_instruction(&mut instruction_data, 50, 20, 5); @@ -276,7 +281,7 @@ fn test_upd_price() { assert_eq!(price_data.prev_slot_, 5); assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.prev_timestamp_, 4); } // Negative prices are accepted @@ -306,7 +311,7 @@ fn test_upd_price() { assert_eq!(price_data.prev_slot_, 5); assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.prev_timestamp_, 4); } } diff --git a/program/rust/src/tests/test_utils.rs b/program/rust/src/tests/test_utils.rs index 853fdcd6..b5f02582 100644 --- a/program/rust/src/tests/test_utils.rs +++ b/program/rust/src/tests/test_utils.rs @@ -134,6 +134,12 @@ pub fn update_clock_slot(clock_account: &mut AccountInfo, slot: u64) { clock_data.to_account_info(clock_account); } +pub fn update_clock_timestamp(clock_account: &mut AccountInfo, timestamp: i64) { + let mut clock_data = clock::Clock::from_account_info(clock_account).unwrap(); + clock_data.unix_timestamp = timestamp; + clock_data.to_account_info(clock_account); +} + impl From for CommandHeader { fn from(val: OracleCommand) -> Self { CommandHeader { From 404f012f20f9f00785db1f3c0c5494dda652cb87 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 8 Apr 2024 13:40:05 +0900 Subject: [PATCH 51/60] address comments --- program/rust/src/tests/test_ema.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index 0206c3d3..dca2b10e 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -34,6 +34,7 @@ use { Serialize, }, solana_program::pubkey::Pubkey, + solana_sdk::account_info::AccountInfo, std::{ fs::File, mem::size_of, @@ -44,7 +45,6 @@ use { #[test] fn test_ema_multiple_publishers_same_slot() -> Result<(), Box> { let mut instruction_data = [0u8; size_of::()]; - populate_instruction(&mut instruction_data, 10, 1, 1); let program_id = Pubkey::new_unique(); @@ -56,11 +56,7 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box(&price_account, PC_VERSION).unwrap(); - price_data.num_ = 1; - price_data.comp_[0].pub_ = *funding_account.key; - } + add_publisher(&mut price_account, funding_account.key, 0); let mut clock_setup = AccountSetup::new_clock(); let mut clock_account = clock_setup.as_account_info(); @@ -69,6 +65,7 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box Result<(), Box(&price_account, PC_VERSION).unwrap(); - price_data.num_ = 2; - price_data.comp_[1].pub_ = *funding_account_two.key; - } + add_publisher(&mut price_account, funding_account_two.key, 1); update_clock_slot(&mut clock_account, 2); @@ -331,3 +324,9 @@ fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_ cmd.publishing_slot = pub_slot; cmd.unused_ = 0; } + +fn add_publisher(price_account: &mut AccountInfo, publisher_key: &Pubkey, index: usize) { + let mut price_data = load_checked::(price_account, PC_VERSION).unwrap(); + price_data.num_ = (index + 1) as u32; + price_data.comp_[index].pub_ = *publisher_key; +} From 7aa7f948d6891dafda7d443d415a60e18d0b7f6d Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 8 Apr 2024 17:09:29 +0900 Subject: [PATCH 52/60] remove print statement --- program/rust/src/tests/test_upd_price.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 1870db37..7effec8b 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -151,7 +151,6 @@ fn test_upd_price() { assert_eq!(price_data.prev_timestamp_, 1); } - println!("test"); // next price doesn't change but slot and timestamp does populate_instruction(&mut instruction_data, 81, 2, 3); update_clock_slot(&mut clock_account, 4); From 144bf94b4e146be6884c9972a23845951b3d0a82 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 8 Apr 2024 23:06:17 +0900 Subject: [PATCH 53/60] add test to simulate program upgrade --- program/rust/src/tests/test_upd_price.rs | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 7effec8b..3678c8ee 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -29,6 +29,7 @@ use { program_error::ProgramError, pubkey::Pubkey, }, + solana_sdk::account_info::AccountInfo, std::mem::size_of, }; @@ -312,6 +313,75 @@ fn test_upd_price() { assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 4); } + + // add new test for multiple publishers and ensure that agg price is not updated multiple times when program upgrade happens in the same slot after the first update + let mut funding_setup_two = AccountSetup::new_funding(); + let funding_account_two = funding_setup_two.as_account_info(); + + add_publisher(&mut price_account, funding_account_two.key, 1); + + populate_instruction(&mut instruction_data, 10, 1, 10); + update_clock_slot(&mut clock_account, 10); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 10); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 10); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 8); + assert_eq!(price_data.agg_.pub_slot_, 10); + assert_eq!(price_data.agg_.price_, 10); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 8); + assert_eq!(price_data.prev_price_, -100); + assert_eq!(price_data.prev_conf_, 1); + assert_eq!(price_data.prev_timestamp_, 4); + } + + // reset twap_.denom_ to 0 to simulate program upgrade in the same slot and make sure agg_.price_ is not updated again + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.prev_twap_.denom_ = 0; + } + populate_instruction(&mut instruction_data, 20, 1, 10); + assert!(process_instruction( + &program_id, + &[ + funding_account_two.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[1].latest_.price_, 20); + assert_eq!(price_data.comp_[1].latest_.conf_, 1); + assert_eq!(price_data.comp_[1].latest_.pub_slot_, 10); + assert_eq!(price_data.comp_[1].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 8); + assert_eq!(price_data.agg_.pub_slot_, 10); + assert_eq!(price_data.agg_.price_, 10); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 8); + assert_eq!(price_data.prev_price_, -100); + assert_eq!(price_data.prev_conf_, 1); + assert_eq!(price_data.prev_timestamp_, 4); + } } // Create an upd_price instruction with the provided parameters @@ -324,3 +394,9 @@ fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_ cmd.publishing_slot = pub_slot; cmd.unused_ = 0; } + +fn add_publisher(price_account: &mut AccountInfo, publisher_key: &Pubkey, index: usize) { + let mut price_data = load_checked::(price_account, PC_VERSION).unwrap(); + price_data.num_ = (index + 1) as u32; + price_data.comp_[index].pub_ = *publisher_key; +} From 6b539db131412e21b0b6676befa1d858844d38fd Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 9 Apr 2024 15:11:43 +0900 Subject: [PATCH 54/60] refactor --- program/rust/src/processor/upd_price.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 99cbbcff..0c0a8351 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -2,7 +2,6 @@ use { crate::{ accounts::{ PriceAccount, - PriceInfo, PythOracleSerialize, UPD_PRICE_WRITE_SEED, }, @@ -129,7 +128,6 @@ pub fn upd_price( let clock = Clock::from_account_info(clock_account)?; let mut publisher_index: usize = 0; - let latest_aggregate_price: PriceInfo; let slots_since_last_update: u64; let noninitial_price_update_after_program_upgrade: bool; @@ -162,10 +160,6 @@ pub fn upd_price( noninitial_price_update_after_program_upgrade = price_data.prev_twap_.denom_ == 0 && slots_since_last_update == 0; - // We assign the current aggregate price to latest_aggregate_price before calling c_upd_aggregate because - // c_upd_aggregate directly modifies the memory of price_data.agg_ to update its values. This is crucial for - // comparisons or operations that rely on the aggregate price state before the update such as slots_since_last_update. - latest_aggregate_price = price_data.agg_; let latest_publisher_price = price_data.comp_[publisher_index].latest_; // Check that publisher is publishing a more recent price @@ -196,7 +190,7 @@ pub fn upd_price( } // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. - if slots_since_last_update > 0 && latest_aggregate_price.status_ == PC_STATUS_TRADING { + if slots_since_last_update > 0 && price_data.agg_.status_ == PC_STATUS_TRADING { price_data.prev_slot_ = price_data.agg_.pub_slot_; price_data.prev_price_ = price_data.agg_.price_; price_data.prev_conf_ = price_data.agg_.conf_; From 5be3e093bf027f7a70bf0b8947d5a3ca7ac5cd53 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 10 Apr 2024 21:32:42 +0900 Subject: [PATCH 55/60] address comments --- program/rust/src/tests/test_ema.rs | 2 ++ program/rust/src/tests/test_upd_price.rs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index dca2b10e..492dfae9 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -126,6 +126,8 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box(&price_account, PC_VERSION).unwrap(); assert_eq!(price_data.prev_twap_.val_, 10); assert_eq!(price_data.prev_twac_.val_, 1); + // The EMA value decreases to 12 despite an increase in the aggregate price due to the higher confidence associated with the new price. + // The EMA calculation considers the weight of 1/confidence for the new price, leading to a lower weighted average when the confidence is high. assert_eq!(price_data.twap_.val_, 12); assert_eq!(price_data.twac_.val_, 1); } diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 3678c8ee..323009a1 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -348,6 +348,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, -100); assert_eq!(price_data.prev_conf_, 1); assert_eq!(price_data.prev_timestamp_, 4); + assert_eq!(price_data.twap_.numer_, 1677311098); + assert_eq!(price_data.twap_.denom_, 1279419481); + assert_eq!(price_data.price_cumulative.price, 86); } // reset twap_.denom_ to 0 to simulate program upgrade in the same slot and make sure agg_.price_ is not updated again @@ -381,6 +384,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, -100); assert_eq!(price_data.prev_conf_, 1); assert_eq!(price_data.prev_timestamp_, 4); + assert_eq!(price_data.twap_.numer_, 1677311098); // twap_.numer_ should not be updated + assert_eq!(price_data.twap_.denom_, 1279419481); // twap_.denom_ should not be updated + assert_eq!(price_data.price_cumulative.price, 86); // price_cumulative should not be updated } } From 458f14371454f44ee8408b7a0a37aa0b1fba3e4b Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 15 Apr 2024 10:57:04 +0900 Subject: [PATCH 56/60] address comments --- program/rust/src/tests/test_ema.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/program/rust/src/tests/test_ema.rs b/program/rust/src/tests/test_ema.rs index 492dfae9..4973ec59 100644 --- a/program/rust/src/tests/test_ema.rs +++ b/program/rust/src/tests/test_ema.rs @@ -56,7 +56,7 @@ fn test_ema_multiple_publishers_same_slot() -> Result<(), Box Result<(), Box(price_account, PC_VERSION).unwrap(); - price_data.num_ = (index + 1) as u32; + let index = price_data.num_ as usize; price_data.comp_[index].pub_ = *publisher_key; + price_data.num_ += 1; } From 49cf2b91503e9898b57627122992345c092caa0f Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 15 Apr 2024 11:11:37 +0900 Subject: [PATCH 57/60] address comments --- program/rust/src/processor/upd_price.rs | 15 ++++++++------- program/rust/src/tests/test_full_publisher_set.rs | 13 ------------- program/rust/src/tests/test_upd_aggregate.rs | 7 ------- 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 0c0a8351..e1440ad6 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -128,7 +128,7 @@ pub fn upd_price( let clock = Clock::from_account_info(clock_account)?; let mut publisher_index: usize = 0; - let slots_since_last_update: u64; + let slots_since_last_successful_aggregate: u64; let noninitial_price_update_after_program_upgrade: bool; // The price_data borrow happens in a scope because it must be @@ -150,15 +150,15 @@ pub fn upd_price( OracleError::PermissionViolation.into(), )?; - // We use last_slot_ to calculate slots_since_last_update. This is because last_slot_ is updated after the aggregate price is updated successfully. - slots_since_last_update = clock.slot - price_data.last_slot_; + // We use last_slot_ to calculate slots_since_last_successful_aggregate. This is because last_slot_ is updated after the aggregate price is updated successfully. + slots_since_last_successful_aggregate = clock.slot - price_data.last_slot_; // Check if the program upgrade has happened in the current slot and aggregate price has been updated, if so, use the old logic to update twap/twac/price_cumulative. // This is to ensure that twap/twac/price_cumulative are calculated correctly during the migration. // We check if prev_twap_.denom_ is == 0 because when the program upgrade has happened, denom_ is initialized to 0 and it can only stay the same or increase while numer_ can be negative if prices are negative. - // And we check if slots_since_last_update == 0 to check if the aggregate price has been updated in the current slot. + // And we check if slots_since_last_successful_aggregate == 0 to check if the aggregate price has been updated in the current slot. noninitial_price_update_after_program_upgrade = - price_data.prev_twap_.denom_ == 0 && slots_since_last_update == 0; + price_data.prev_twap_.denom_ == 0 && slots_since_last_successful_aggregate == 0; let latest_publisher_price = price_data.comp_[publisher_index].latest_; @@ -190,7 +190,8 @@ pub fn upd_price( } // If the price update is the first in the slot and the aggregate is trading, update the previous slot, price, conf, and timestamp. - if slots_since_last_update > 0 && price_data.agg_.status_ == PC_STATUS_TRADING { + if slots_since_last_successful_aggregate > 0 && price_data.agg_.status_ == PC_STATUS_TRADING + { price_data.prev_slot_ = price_data.agg_.pub_slot_; price_data.prev_price_ = price_data.agg_.price_; price_data.prev_conf_ = price_data.agg_.conf_; @@ -224,7 +225,7 @@ pub fn upd_price( // from the last successful aggregated price update as their basis for recalculation. This // ensures that each update within a slot builds upon the last and not the twap/twac/price_cumulative // that is calculated right after the publishers' individual price updates. - if slots_since_last_update > 0 { + if slots_since_last_successful_aggregate > 0 { price_data.prev_twap_ = price_data.twap_; price_data.prev_twac_ = price_data.twac_; price_data.prev_price_cumulative = price_data.price_cumulative; diff --git a/program/rust/src/tests/test_full_publisher_set.rs b/program/rust/src/tests/test_full_publisher_set.rs index bc4cfd13..87f48b71 100644 --- a/program/rust/src/tests/test_full_publisher_set.rs +++ b/program/rust/src/tests/test_full_publisher_set.rs @@ -36,7 +36,6 @@ async fn test_full_publisher_set() -> Result<(), Box> { .await; let price = price_accounts["LTC"]; - let n_pubs = pub_keypairs.len(); // Divide publishers into two even parts (assuming the max PC_NUM_COMP size is even) @@ -61,18 +60,6 @@ async fn test_full_publisher_set() -> Result<(), Box> { sim.upd_price(second_kp, price, second_quote).await?; } - // Advance slot once from 1 to 2 - sim.warp_to_slot(2).await?; - - // Final price update to trigger aggregation - let first_kp = pub_keypairs.first().unwrap(); - let first_quote = Quote { - price: 100, - confidence: 30, - status: PC_STATUS_TRADING, - }; - sim.upd_price(first_kp, price, first_quote).await?; - { let price_data = sim .get_account_data_as::(price) diff --git a/program/rust/src/tests/test_upd_aggregate.rs b/program/rust/src/tests/test_upd_aggregate.rs index 7bb297c3..657c6cd6 100644 --- a/program/rust/src/tests/test_upd_aggregate.rs +++ b/program/rust/src/tests/test_upd_aggregate.rs @@ -115,13 +115,6 @@ fn test_upd_aggregate() { price_data.timestamp_ = 0; price_data.agg_.pub_slot_ = 1000; price_data.comp_[0].latest_ = p1; - price_data.prev_slot_ = 0; - price_data.prev_price_ = 0; - price_data.prev_conf_ = 0; - price_data.prev_timestamp_ = 0; - price_data.agg_.price_ = 0; - price_data.agg_.conf_ = 0; - price_data.agg_.status_ = PC_STATUS_UNKNOWN; } unsafe { From 490e6589df115a080323ee6b6c98ad74312cfe30 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 15 Apr 2024 12:27:28 +0900 Subject: [PATCH 58/60] address comments --- program/rust/src/tests/test_upd_price.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 323009a1..70ade80c 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -80,6 +80,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); + assert_eq!(price_data.last_slot_, 1); assert_eq!(price_data.agg_.pub_slot_, 1); assert_eq!(price_data.agg_.price_, 42); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -112,6 +113,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 0); + assert_eq!(price_data.last_slot_, 1); assert_eq!(price_data.agg_.pub_slot_, 1); assert_eq!(price_data.agg_.price_, 42); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -143,6 +145,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 2); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 1); + assert_eq!(price_data.last_slot_, 3); assert_eq!(price_data.agg_.pub_slot_, 3); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -174,13 +177,14 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 3); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 3); + assert_eq!(price_data.last_slot_, 4); assert_eq!(price_data.agg_.pub_slot_, 4); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); assert_eq!(price_data.prev_slot_, 3); assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); - assert_eq!(price_data.timestamp_, 4); // We only check for timestamp_ here because test_upd_price doesn't directly update timestamp_, this is updated through c_upd_aggregate which is tested in test_upd_aggregate, but we assert here to show that in subsequent asserts for prev_tim + assert_eq!(price_data.timestamp_, 4); // We only check for timestamp_ here because test_upd_price doesn't directly update timestamp_, this is updated through c_upd_aggregate which is tested in test_upd_aggregate, but we assert here to show that in subsequent asserts for prev_timestamp_ the value should be updated to this value assert_eq!(price_data.prev_timestamp_, 1); } @@ -205,6 +209,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 4); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 4); + assert_eq!(price_data.last_slot_, 5); assert_eq!(price_data.agg_.pub_slot_, 5); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -237,6 +242,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 4); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 4); + assert_eq!(price_data.last_slot_, 5); assert_eq!(price_data.agg_.pub_slot_, 5); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -275,6 +281,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 5); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); assert_eq!(price_data.valid_slot_, 5); + assert_eq!(price_data.last_slot_, 5); assert_eq!(price_data.agg_.pub_slot_, 6); assert_eq!(price_data.agg_.price_, 81); assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); @@ -305,6 +312,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 7); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 6); + assert_eq!(price_data.last_slot_, 8); assert_eq!(price_data.agg_.pub_slot_, 8); assert_eq!(price_data.agg_.price_, -100); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -341,6 +349,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[0].latest_.pub_slot_, 10); assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 8); + assert_eq!(price_data.last_slot_, 10); assert_eq!(price_data.agg_.pub_slot_, 10); assert_eq!(price_data.agg_.price_, 10); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); @@ -377,6 +386,7 @@ fn test_upd_price() { assert_eq!(price_data.comp_[1].latest_.pub_slot_, 10); assert_eq!(price_data.comp_[1].latest_.status_, PC_STATUS_TRADING); assert_eq!(price_data.valid_slot_, 8); + assert_eq!(price_data.last_slot_, 10); assert_eq!(price_data.agg_.pub_slot_, 10); assert_eq!(price_data.agg_.price_, 10); assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); From 0d404c34bf1f250ed8726ab25b794be1d2e6ed84 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 15 Apr 2024 13:14:13 +0900 Subject: [PATCH 59/60] merge test_upd_price with test_upd_price_v2 --- program/rust/src/tests/mod.rs | 5 +- program/rust/src/tests/test_upd_price.rs | 151 +++++- program/rust/src/tests/test_upd_price_v2.rs | 518 -------------------- 3 files changed, 147 insertions(+), 527 deletions(-) delete mode 100644 program/rust/src/tests/test_upd_price_v2.rs diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index 8208d153..fdfa8e70 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -19,13 +19,10 @@ mod test_publish_batch; mod test_set_max_latency; mod test_set_min_pub; mod test_sizes; +mod test_twap; mod test_upd_aggregate; mod test_upd_permissions; mod test_upd_price; mod test_upd_price_no_fail_on_error; mod test_upd_product; mod test_utils; - - -mod test_twap; -mod test_upd_price_v2; diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 70ade80c..915af345 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -88,6 +88,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 0); assert_eq!(price_data.prev_conf_, 0); assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.price_cumulative.price, 42); + assert_eq!(price_data.price_cumulative.conf, 2); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // add some prices for current slot - get rejected @@ -121,6 +124,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 0); assert_eq!(price_data.prev_conf_, 0); assert_eq!(price_data.prev_timestamp_, 0); + assert_eq!(price_data.price_cumulative.price, 42); + assert_eq!(price_data.price_cumulative.conf, 2); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // update new price in new slot, aggregate should be updated and prev values should be updated @@ -153,6 +159,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 42); assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 1); + assert_eq!(price_data.price_cumulative.price, 42 + 2 * 81); + assert_eq!(price_data.price_cumulative.conf, 3 * 2); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // next price doesn't change but slot and timestamp does @@ -186,6 +195,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.timestamp_, 4); // We only check for timestamp_ here because test_upd_price doesn't directly update timestamp_, this is updated through c_upd_aggregate which is tested in test_upd_aggregate, but we assert here to show that in subsequent asserts for prev_timestamp_ the value should be updated to this value assert_eq!(price_data.prev_timestamp_, 1); + assert_eq!(price_data.price_cumulative.price, 42 + 3 * 81); + assert_eq!(price_data.price_cumulative.conf, 4 * 2); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // next price doesn't change and neither does aggregate but slot does @@ -217,6 +229,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 4); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 5 * 2); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // try to publish back-in-time @@ -250,6 +265,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 4); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 5 * 2); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } populate_instruction(&mut instruction_data, 50, 20, 5); @@ -289,6 +307,9 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 4); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); + assert_eq!(price_data.price_cumulative.conf, 2 * 5); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // Negative prices are accepted @@ -320,13 +341,16 @@ fn test_upd_price() { assert_eq!(price_data.prev_price_, 81); assert_eq!(price_data.prev_conf_, 2); assert_eq!(price_data.prev_timestamp_, 4); + assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 3); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 3); // 2 * 5 + 1 * 3 + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // add new test for multiple publishers and ensure that agg price is not updated multiple times when program upgrade happens in the same slot after the first update let mut funding_setup_two = AccountSetup::new_funding(); let funding_account_two = funding_setup_two.as_account_info(); - add_publisher(&mut price_account, funding_account_two.key, 1); + add_publisher(&mut price_account, funding_account_two.key); populate_instruction(&mut instruction_data, 10, 1, 10); update_clock_slot(&mut clock_account, 10); @@ -359,7 +383,12 @@ fn test_upd_price() { assert_eq!(price_data.prev_timestamp_, 4); assert_eq!(price_data.twap_.numer_, 1677311098); assert_eq!(price_data.twap_.denom_, 1279419481); - assert_eq!(price_data.price_cumulative.price, 86); + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + ); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 5); // 2 * 5 + 1 * 5 + assert_eq!(price_data.price_cumulative.num_down_slots, 0); } // reset twap_.denom_ to 0 to simulate program upgrade in the same slot and make sure agg_.price_ is not updated again @@ -396,7 +425,111 @@ fn test_upd_price() { assert_eq!(price_data.prev_timestamp_, 4); assert_eq!(price_data.twap_.numer_, 1677311098); // twap_.numer_ should not be updated assert_eq!(price_data.twap_.denom_, 1279419481); // twap_.denom_ should not be updated - assert_eq!(price_data.price_cumulative.price, 86); // price_cumulative should not be updated + + // price_cumulative should not be updated + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + ); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 5); + assert_eq!(price_data.price_cumulative.num_down_slots, 0); + } + + remove_publisher(&mut price_account); + // Big gap + populate_instruction(&mut instruction_data, 60, 4, 50); + update_clock_slot(&mut clock_account, 50); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 60); + assert_eq!(price_data.comp_[0].latest_.conf_, 4); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 50); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 10); + assert_eq!(price_data.agg_.pub_slot_, 50); + assert_eq!(price_data.agg_.price_, 60); + assert_eq!(price_data.agg_.conf_, 4); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + ); + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 5 + 40 * 4); + assert_eq!(price_data.price_cumulative.num_down_slots, 15); + } + + // add new test for multiple publishers and ensure that price_cumulative is updated correctly + add_publisher(&mut price_account, funding_account_two.key); + populate_instruction(&mut instruction_data, 10, 1, 100); + update_clock_slot(&mut clock_account, 100); + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + populate_instruction(&mut instruction_data, 20, 2, 100); + assert!(process_instruction( + &program_id, + &[ + funding_account_two.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 10); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 100); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.comp_[1].latest_.price_, 20); + assert_eq!(price_data.comp_[1].latest_.conf_, 2); + assert_eq!(price_data.comp_[1].latest_.pub_slot_, 100); + assert_eq!(price_data.comp_[1].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 100); + assert_eq!(price_data.agg_.pub_slot_, 100); + assert_eq!(price_data.agg_.price_, 14); + assert_eq!(price_data.agg_.conf_, 6); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 50); + assert_eq!(price_data.prev_price_, 60); + assert_eq!(price_data.prev_conf_, 4); + assert_eq!( + price_data.prev_price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + ); + assert_eq!(price_data.prev_price_cumulative.conf, 2 * 5 + 5 + 40 * 4); + assert_eq!(price_data.prev_price_cumulative.num_down_slots, 15); + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + 14 * 50 + ); // (42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + 55) is the price cumulative from the previous test and 14 * 49 (slot_gap) is the price cumulative from this test + assert_eq!( + price_data.price_cumulative.conf, + 2 * 5 + 5 + 40 * 4 + 6 * 50 + ); + assert_eq!(price_data.price_cumulative.num_down_slots, 40); // prev num_down_slots was 15 and since pub slot is 100 and last pub slot was 50, slot_gap is 50 and default latency is 25, so num_down_slots = 50 - 25 = 25, so total num_down_slots = 15 + 25 = 40 } } @@ -411,8 +544,16 @@ fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_ cmd.unused_ = 0; } -fn add_publisher(price_account: &mut AccountInfo, publisher_key: &Pubkey, index: usize) { +fn add_publisher(price_account: &mut AccountInfo, publisher_key: &Pubkey) { let mut price_data = load_checked::(price_account, PC_VERSION).unwrap(); - price_data.num_ = (index + 1) as u32; + let index = price_data.num_ as usize; price_data.comp_[index].pub_ = *publisher_key; + price_data.num_ += 1; +} + +fn remove_publisher(price_account: &mut AccountInfo) { + let mut price_data = load_checked::(price_account, PC_VERSION).unwrap(); + if price_data.num_ > 0 { + price_data.num_ -= 1; + } } diff --git a/program/rust/src/tests/test_upd_price_v2.rs b/program/rust/src/tests/test_upd_price_v2.rs deleted file mode 100644 index a2e326a0..00000000 --- a/program/rust/src/tests/test_upd_price_v2.rs +++ /dev/null @@ -1,518 +0,0 @@ -use { - crate::{ - accounts::{ - PriceAccount, - PythAccount, - }, - c_oracle_header::{ - PC_STATUS_IGNORED, - PC_STATUS_TRADING, - PC_STATUS_UNKNOWN, - PC_VERSION, - }, - deserialize::{ - load_checked, - load_mut, - }, - instruction::{ - OracleCommand, - UpdPriceArgs, - }, - processor::process_instruction, - tests::test_utils::{ - update_clock_slot, - AccountSetup, - }, - }, - solana_program::{ - program_error::ProgramError, - pubkey::Pubkey, - }, - std::mem::size_of, -}; - -#[test] -fn test_upd_price_v2() -> Result<(), Box> { - let mut instruction_data = [0u8; size_of::()]; - populate_instruction(&mut instruction_data, 42, 2, 1); - - let program_id = Pubkey::new_unique(); - - let mut funding_setup = AccountSetup::new_funding(); - let funding_account = funding_setup.as_account_info(); - - let mut price_setup = AccountSetup::new::(&program_id); - let mut price_account = price_setup.as_account_info(); - price_account.is_signer = false; - PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); - - { - let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - price_data.num_ = 1; - price_data.comp_[0].pub_ = *funding_account.key; - } - - let mut clock_setup = AccountSetup::new_clock(); - let mut clock_account = clock_setup.as_account_info(); - clock_account.is_signer = false; - clock_account.is_writable = false; - - update_clock_slot(&mut clock_account, 1); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 42); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 0); - assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 42); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42); - assert_eq!(price_data.price_cumulative.conf, 2); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // add some prices for current slot - get rejected - populate_instruction(&mut instruction_data, 43, 2, 1); - - assert_eq!( - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone() - ], - &instruction_data - ), - Err(ProgramError::InvalidArgument) - ); - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 42); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 0); - assert_eq!(price_data.agg_.pub_slot_, 1); - assert_eq!(price_data.agg_.price_, 42); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42); - assert_eq!(price_data.price_cumulative.conf, 2); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // add next price in new slot triggering snapshot and aggregate calc - populate_instruction(&mut instruction_data, 81, 2, 2); - update_clock_slot(&mut clock_account, 3); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 81); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 2); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 1); - assert_eq!(price_data.agg_.pub_slot_, 3); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42 + 2 * 81); - assert_eq!(price_data.price_cumulative.conf, 3 * 2); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // next price doesn't change but slot does - populate_instruction(&mut instruction_data, 81, 2, 3); - update_clock_slot(&mut clock_account, 4); - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 81); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 3); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 3); - assert_eq!(price_data.agg_.pub_slot_, 4); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42 + 3 * 81); - assert_eq!(price_data.price_cumulative.conf, 4 * 2); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // next price doesn't change and neither does aggregate but slot does - populate_instruction(&mut instruction_data, 81, 2, 4); - update_clock_slot(&mut clock_account, 5); - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 81); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 4); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 4); - assert_eq!(price_data.agg_.pub_slot_, 5); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); - assert_eq!(price_data.price_cumulative.conf, 5 * 2); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // try to publish back-in-time - populate_instruction(&mut instruction_data, 81, 2, 1); - update_clock_slot(&mut clock_account, 5); - assert_eq!( - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone() - ], - &instruction_data - ), - Err(ProgramError::InvalidArgument) - ); - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 81); - assert_eq!(price_data.comp_[0].latest_.conf_, 2); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 4); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 4); - assert_eq!(price_data.agg_.pub_slot_, 5); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); - assert_eq!(price_data.price_cumulative.conf, 5 * 2); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - populate_instruction(&mut instruction_data, 50, 20, 5); - update_clock_slot(&mut clock_account, 6); - - // Publishing a wide CI results in a status of unknown. - - // check that someone doesn't accidentally break the test. - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - } - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 50); - assert_eq!(price_data.comp_[0].latest_.conf_, 20); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 5); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); - assert_eq!(price_data.valid_slot_, 5); - assert_eq!(price_data.agg_.pub_slot_, 6); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); - assert_eq!(price_data.price_cumulative.conf, 2 * 5); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // Crank one more time and aggregate should be unknown - populate_instruction(&mut instruction_data, 50, 20, 6); - update_clock_slot(&mut clock_account, 7); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 50); - assert_eq!(price_data.comp_[0].latest_.conf_, 20); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 6); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); - assert_eq!(price_data.valid_slot_, 6); - assert_eq!(price_data.agg_.pub_slot_, 7); - assert_eq!(price_data.agg_.price_, 81); - assert_eq!(price_data.agg_.conf_, 2); - assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); - assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4); - assert_eq!(price_data.price_cumulative.conf, 2 * 5); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // Negative prices are accepted - populate_instruction(&mut instruction_data, -100, 1, 7); - update_clock_slot(&mut clock_account, 8); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, -100); - assert_eq!(price_data.comp_[0].latest_.conf_, 1); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 7); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 7); - assert_eq!(price_data.agg_.pub_slot_, 8); - assert_eq!(price_data.agg_.price_, -100); - assert_eq!(price_data.agg_.conf_, 1); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 3); - assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 3); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // Crank again for aggregate - populate_instruction(&mut instruction_data, -100, 1, 8); - update_clock_slot(&mut clock_account, 9); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, -100); - assert_eq!(price_data.comp_[0].latest_.conf_, 1); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 8); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 8); - assert_eq!(price_data.agg_.pub_slot_, 9); - assert_eq!(price_data.agg_.price_, -100); - assert_eq!(price_data.agg_.conf_, 1); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 4); - assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 4); - assert_eq!(price_data.price_cumulative.num_down_slots, 0); - } - - // Big gap - - populate_instruction(&mut instruction_data, 60, 4, 50); - update_clock_slot(&mut clock_account, 50); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 60); - assert_eq!(price_data.comp_[0].latest_.conf_, 4); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 50); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 9); - assert_eq!(price_data.agg_.pub_slot_, 50); - assert_eq!(price_data.agg_.price_, 60); - assert_eq!(price_data.agg_.conf_, 4); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!( - price_data.price_cumulative.price, - 42 + 81 * 4 - 100 * 4 + 60 * 41 - ); - assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 4 + 41 * 4); - assert_eq!(price_data.price_cumulative.num_down_slots, 16); - } - - // Crank again for aggregate - - populate_instruction(&mut instruction_data, 55, 5, 51); - update_clock_slot(&mut clock_account, 51); - - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 55); - assert_eq!(price_data.comp_[0].latest_.conf_, 5); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 51); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 50); - assert_eq!(price_data.agg_.pub_slot_, 51); - assert_eq!(price_data.agg_.price_, 55); - assert_eq!(price_data.agg_.conf_, 5); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!( - price_data.price_cumulative.price, - 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 - ); - assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 4 + 41 * 4 + 5); - assert_eq!(price_data.price_cumulative.num_down_slots, 16); - } - - // add new test for multiple publishers and ensure that price_cumulative is updated correctly - let mut funding_setup_two = AccountSetup::new_funding(); - let funding_account_two = funding_setup_two.as_account_info(); - - { - let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - price_data.num_ = 2; - price_data.comp_[1].pub_ = *funding_account_two.key; - } - - populate_instruction(&mut instruction_data, 10, 1, 100); - update_clock_slot(&mut clock_account, 100); - process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - populate_instruction(&mut instruction_data, 20, 2, 100); - process_instruction( - &program_id, - &[ - funding_account_two.clone(), - price_account.clone(), - clock_account.clone(), - ], - &instruction_data, - )?; - - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.comp_[0].latest_.price_, 10); - assert_eq!(price_data.comp_[0].latest_.conf_, 1); - assert_eq!(price_data.comp_[0].latest_.pub_slot_, 100); - assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.comp_[1].latest_.price_, 20); - assert_eq!(price_data.comp_[1].latest_.conf_, 2); - assert_eq!(price_data.comp_[1].latest_.pub_slot_, 100); - assert_eq!(price_data.comp_[1].latest_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.valid_slot_, 100); - assert_eq!(price_data.agg_.pub_slot_, 100); - assert_eq!(price_data.agg_.price_, 14); - assert_eq!(price_data.agg_.conf_, 6); - assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); - assert_eq!(price_data.prev_slot_, 51); - assert_eq!(price_data.prev_price_, 55); - assert_eq!(price_data.prev_conf_, 5); - assert_eq!( - price_data.prev_price_cumulative.price, - 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 - ); - assert_eq!( - price_data.prev_price_cumulative.conf, - 2 * 5 + 4 + 41 * 4 + 5 - ); - assert_eq!(price_data.prev_price_cumulative.num_down_slots, 16); - assert_eq!( - price_data.price_cumulative.price, - 42 + 81 * 4 - 100 * 4 + 60 * 41 + 55 + 14 * 49 - ); // (42 + 81 * 4 - 100 * 4 + 60 * 41 + 55) is the price cumulative from the previous test and 14 * 49 (slot_gap) is the price cumulative from this test - assert_eq!( - price_data.price_cumulative.conf, - 2 * 5 + 4 + 41 * 4 + 5 + 6 * 49 - ); - assert_eq!(price_data.price_cumulative.num_down_slots, 40); // prev num_down_slots was 16 and since pub slot is 100 and last pub slot was 51, slot_gap is 49 and default latency is 25, so num_down_slots = 49 - 25 = 24, so total num_down_slots = 16 + 24 = 40 - } - - Ok(()) -} - -// Create an upd_price instruction with the provided parameters -fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_slot: u64) { - let mut cmd = load_mut::(instruction_data).unwrap(); - cmd.header = OracleCommand::UpdPrice.into(); - cmd.status = PC_STATUS_TRADING; - cmd.price = price; - cmd.confidence = conf; - cmd.publishing_slot = pub_slot; - cmd.unused_ = 0; -} From 9c3a9a7132768e7b72cc5b4cf0dbbc4317464c3f Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 16 Apr 2024 12:55:47 +0900 Subject: [PATCH 60/60] add asserts --- program/rust/src/tests/test_upd_price.rs | 30 +++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/program/rust/src/tests/test_upd_price.rs b/program/rust/src/tests/test_upd_price.rs index 915af345..d370030f 100644 --- a/program/rust/src/tests/test_upd_price.rs +++ b/program/rust/src/tests/test_upd_price.rs @@ -485,6 +485,34 @@ fn test_upd_price() { ) .is_ok()); + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 10); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 100); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 50); + assert_eq!(price_data.agg_.pub_slot_, 100); + assert_eq!(price_data.agg_.price_, 10); + assert_eq!(price_data.agg_.conf_, 1); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.prev_slot_, 50); + assert_eq!(price_data.prev_price_, 60); + assert_eq!(price_data.prev_conf_, 4); + assert_eq!( + price_data.prev_price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + ); + assert_eq!(price_data.prev_price_cumulative.conf, 2 * 5 + 5 + 40 * 4); + assert_eq!(price_data.prev_price_cumulative.num_down_slots, 15); + assert_eq!( + price_data.price_cumulative.price, + 42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + 10 * 50 + ); // (42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40) is the price cumulative from the previous test and 10 * 50 (slot_gap) is the price cumulative from this test + assert_eq!(price_data.price_cumulative.conf, 2 * 5 + 5 + 40 * 4 + 50); + assert_eq!(price_data.price_cumulative.num_down_slots, 40); // prev num_down_slots was 15 and since pub slot is 100 and last pub slot was 50, slot_gap is 50 and default latency is 25, so num_down_slots = 50 - 25 = 25, so total num_down_slots = 15 + 25 = 40 + } + populate_instruction(&mut instruction_data, 20, 2, 100); assert!(process_instruction( &program_id, @@ -524,7 +552,7 @@ fn test_upd_price() { assert_eq!( price_data.price_cumulative.price, 42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + 14 * 50 - ); // (42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40 + 55) is the price cumulative from the previous test and 14 * 49 (slot_gap) is the price cumulative from this test + ); // (42 + 81 * 4 - 100 * 3 + 10 * 2 + 60 * 40) is the price cumulative from the previous test and 14 * 50 (slot_gap) is the price cumulative from this test assert_eq!( price_data.price_cumulative.conf, 2 * 5 + 5 + 40 * 4 + 6 * 50