From 86bf3532d5a9f93e0c9d2409edb906520fe3f731 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 22 Mar 2024 14:57:10 +0900 Subject: [PATCH 01/13] remove solana specific code --- .github/workflows/docker.yaml | 10 --- README.md | 4 +- program/rust/Cargo.toml | 1 + program/rust/src/accounts.rs | 4 +- program/rust/src/accounts/price.rs | 77 ------------------------ program/rust/src/processor/upd_price.rs | 31 +++------- program/rust/src/tests/mod.rs | 3 +- program/rust/src/tests/pyth_simulator.rs | 7 +-- program/rust/src/tests/test_sizes.rs | 19 ------ scripts/build-bpf.sh | 7 --- 10 files changed, 14 insertions(+), 149 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 06b57dc4f..d05dac421 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -61,17 +61,7 @@ jobs: run : | docker create -ti --name container "${DOCKER_IMAGE}" bash docker cp container:/home/pyth/pyth-client/target/pyth/pythnet/pyth_oracle_pythnet.so . - docker cp container:/home/pyth/pyth-client/target/pyth/solana/pyth_oracle_solana.so . docker rm -f container - - - name : Publish Solana binary - if : env.IS_ORACLE_RELEASE == 'true' - uses: svenstaro/upload-release-action@133984371c30d34e38222a64855679a414cb7575 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ./pyth_oracle_solana.so - asset_name: pyth_oracle_solana.so - tag: ${{ github.ref }} - name : Publish Pythnet binary if : env.IS_ORACLE_RELEASE == 'true' diff --git a/README.md b/README.md index 46e9a924d..6accaccaa 100644 --- a/README.md +++ b/README.md @@ -155,10 +155,10 @@ Finally, in docker extension inside VS Code click right and choose "Attach VS Co Oracle program upgrades are managed by the Pythian Council multisig. The steps to deploy a new version are: -1. Create a release branch from `main`. This should include binaries for both the Solana mainnet and Pythnet oracle programs (`pyth_oracle_solana.so` and `pyth_oracle_pythnet.so`). +1. Create a release branch from `main`. This should include binary for both the Pythnet oracle programs (`pyth_oracle_pythnet.so`). 2. [Install Solana CLI](https://docs.solana.com/cli/install-solana-cli-tools#use-solanas-install-tool) if not already installed. 3. Set Solana config for the target network, e.g., devnet: `solana config set --url https://api.devnet.solana.com`. -4. Execute `solana program write-buffer ` where `` can be `pyth_oracle_solana.so` or `pyth_oracle_pythnet.so` to obtain the buffer address. +4. Execute `solana program write-buffer pyth_oracle_pythnet.so` to obtain the buffer address. 5. Run `solana program show ` to obtain the authority of the current program. 6. Use `solana program set-buffer-authority --new-buffer-authority ` to assign the upgrade authority from the previous step to the buffer address. 7. Submit a proposal with [`xc-admin`](https://github.com/pyth-network/pyth-crosschain/tree/main/governance/xc_admin/packages/xc_admin_cli) for program upgrade using the `upgrade-program` command. diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 2b88bdab9..1c2ff744e 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -43,6 +43,7 @@ lazy_static = "1.4.0" # to cargo-build-bpf at that point - we manually capture them at # compile-time and pass on to the child process. [features] +default = ["pythnet"] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] library = [] diff --git a/program/rust/src/accounts.rs b/program/rust/src/accounts.rs index 07cb5a051..5958fc88c 100644 --- a/program/rust/src/accounts.rs +++ b/program/rust/src/accounts.rs @@ -43,8 +43,6 @@ mod product; // Some types only exist during use as a library. #[cfg(feature = "strum")] pub use price::MessageType; -#[cfg(feature = "pythnet")] -pub use price::PriceCumulative; #[cfg(test)] pub use product::{ account_has_key_values, @@ -56,6 +54,7 @@ pub use { price::{ PriceAccount, PriceComponent, + PriceCumulative, PriceEma, PriceInfo, PythOracleSerialize, @@ -72,7 +71,6 @@ pub use { /// are authorized to perform certain administrative actions. pub const PERMISSIONS_SEED: &str = "permissions"; -#[cfg(feature = "pythnet")] /// The update price instruction can optionally invoke another program via CPI. The /// CPI will be signed with the PDA `[UPD_PRICE_WRITE_SEED, invoked_program_public_key]` /// such that the caller can authenticate its origin. diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index f69926b5a..421cc827b 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -1,7 +1,4 @@ -#[cfg(feature = "pythnet")] pub use price_pythnet::*; -#[cfg(not(feature = "pythnet"))] -pub use price_solana::*; use { super::{ AccountHeader, @@ -21,7 +18,6 @@ use { }; /// Pythnet-specific PriceAccount implementation -#[cfg(feature = "pythnet")] mod price_pythnet { pub type PriceAccount = PriceAccountPythnet; @@ -191,79 +187,6 @@ mod price_pythnet { } } -/// Solana-specific PriceAccount implementation -#[cfg(not(feature = "pythnet"))] -mod price_solana { - pub type PriceAccount = PriceAccountSolana; - - use { - super::*, - crate::c_oracle_header::PC_NUM_COMP_SOLANA, - }; - - /// Legacy Solana-only price account format. This price account - /// schema is maintained for compatibility reasons. - /// - /// The main source of incompatibility on mainnet/devnet is the - /// possibility of strict account size checks which could break - /// existing users if they encountered a longer struct like - /// PriceAccountPythnet (see mod price_pythnet for details) - #[repr(C)] - #[derive(Copy, Clone, Pod, Zeroable)] - pub struct PriceAccountSolana { - pub header: AccountHeader, - /// Type of the price account - pub price_type: u32, - /// Exponent for the published prices - pub exponent: i32, - /// Current number of authorized publishers - pub num_: u32, - /// Number of valid quotes for the last aggregation - pub num_qt_: u32, - /// Last slot with a succesful aggregation (status : TRADING) - pub last_slot_: u64, - /// Second to last slot where aggregation was attempted - pub valid_slot_: u64, - /// Ema for price - pub twap_: PriceEma, - /// Ema for confidence - pub twac_: PriceEma, - /// Last time aggregation was attempted - pub timestamp_: i64, - /// Minimum valid publisher quotes for a succesful aggregation - pub min_pub_: u8, - /// Whether the current aggregate price has been sent as a message to the message buffer. - /// 0 = false, 1 = true. (this is a u8 to make the Pod trait happy) - pub message_sent_: u8, - /// Configurable max latency in slots between send and receive - pub max_latency_: u8, - /// Unused placeholder for alignment - pub unused_2_: i8, - pub unused_3_: i32, - /// Corresponding product account - pub product_account: Pubkey, - /// Next price account in the list - pub next_price_account: Pubkey, - /// Second to last slot where aggregation was succesful (i.e. status : TRADING) - pub prev_slot_: u64, - /// Aggregate price at prev_slot_ - pub prev_price_: i64, - /// Confidence interval at prev_slot_ - pub prev_conf_: u64, - /// Timestamp of prev_slot_ - pub prev_timestamp_: i64, - /// Last attempted aggregate results - pub agg_: PriceInfo, - /// Publishers' price components - pub comp_: [PriceComponent; PC_NUM_COMP_SOLANA as usize], - } - - impl PythAccount for PriceAccountSolana { - const ACCOUNT_TYPE: u32 = PC_ACCTYPE_PRICE; - const INITIAL_SIZE: u32 = size_of::() as u32; - } -} - #[repr(C)] #[derive(Copy, Clone, Pod, Zeroable)] pub struct PriceComponent { diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 4af6e89a4..524201239 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -1,20 +1,10 @@ -#[cfg(feature = "pythnet")] -use { - crate::accounts::{ - PythOracleSerialize, - UPD_PRICE_WRITE_SEED, - }, - solana_program::instruction::{ - AccountMeta, - Instruction, - }, - solana_program::program::invoke_signed, -}; use { crate::{ accounts::{ PriceAccount, PriceInfo, + PythOracleSerialize, + UPD_PRICE_WRITE_SEED, }, deserialize::{ load, @@ -35,6 +25,11 @@ use { account_info::AccountInfo, clock::Clock, entrypoint::ProgramResult, + instruction::{ + AccountMeta, + Instruction, + }, + program::invoke_signed, program_error::ProgramError, pubkey::Pubkey, sysvar::Sysvar, @@ -44,10 +39,7 @@ use { #[cfg(target_arch = "bpf")] #[link(name = "cpyth-bpf")] extern "C" { - #[cfg(feature = "pythnet")] pub fn c_upd_aggregate_pythnet(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; - #[cfg(not(feature = "pythnet"))] - pub fn c_upd_aggregate_solana(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; #[allow(unused)] pub fn c_upd_twap(_input: *mut u8, nslots: i64); @@ -56,10 +48,7 @@ extern "C" { #[cfg(not(target_arch = "bpf"))] #[link(name = "cpyth-native")] extern "C" { - #[cfg(feature = "pythnet")] pub fn c_upd_aggregate_pythnet(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; - #[cfg(not(feature = "pythnet"))] - pub fn c_upd_aggregate_solana(_input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool; #[allow(unused)] pub fn c_upd_twap(_input: *mut u8, nslots: i64); @@ -67,10 +56,7 @@ extern "C" { #[inline] pub unsafe fn c_upd_aggregate(input: *mut u8, clock_slot: u64, clock_timestamp: i64) -> bool { - #[cfg(feature = "pythnet")] - return c_upd_aggregate_pythnet(input, clock_slot, clock_timestamp); - #[cfg(not(feature = "pythnet"))] - return c_upd_aggregate_solana(input, clock_slot, clock_timestamp); + c_upd_aggregate_pythnet(input, clock_slot, clock_timestamp) } /// Publish component price, never returning an error even if the update failed @@ -208,7 +194,6 @@ pub fn upd_price( let mut price_data = load_checked::(price_account, cmd_args.header.version)?; // Feature-gated accumulator-specific code, used only on pythnet/pythtest - #[cfg(feature = "pythnet")] { // We want to send a message every time the aggregate price updates. However, during the migration, // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index 1263ae604..8208d1536 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -26,7 +26,6 @@ mod test_upd_price_no_fail_on_error; mod test_upd_product; mod test_utils; -#[cfg(feature = "pythnet")] + mod test_twap; -#[cfg(feature = "pythnet")] mod test_upd_price_v2; diff --git a/program/rust/src/tests/pyth_simulator.rs b/program/rust/src/tests/pyth_simulator.rs index 6a77ba0c7..2c10a6ab0 100644 --- a/program/rust/src/tests/pyth_simulator.rs +++ b/program/rust/src/tests/pyth_simulator.rs @@ -78,11 +78,7 @@ lazy_static::lazy_static! { // Detect features and pass them onto cargo-build-bpf. // IMPORTANT: All features of this crate must have gates added to this vector. - let features: Vec<&str> = vec![ - #[cfg(feature = "pythnet")] - "pythnet", - - ]; + let features: Vec<&str> = vec!["pythnet"]; let mut cmd = std::process::Command::new("cargo"); cmd.arg("build-bpf"); @@ -215,7 +211,6 @@ impl PythSimulator { .await .unwrap(); - result } diff --git a/program/rust/src/tests/test_sizes.rs b/program/rust/src/tests/test_sizes.rs index 153e82356..9ec1e37b2 100644 --- a/program/rust/src/tests/test_sizes.rs +++ b/program/rust/src/tests/test_sizes.rs @@ -53,7 +53,6 @@ fn test_sizes() { size_of::() + 2 * size_of::() ); - #[cfg(feature = "pythnet")] { use crate::{ accounts::PriceCumulative, @@ -77,24 +76,6 @@ fn test_sizes() { assert_eq!(size_of::(), 48); } - #[cfg(not(feature = "pythnet"))] - { - use crate::c_oracle_header::PC_NUM_COMP_SOLANA; - - // Sanity-check the Solana PC_NUM_COMP - assert_eq!(PC_NUM_COMP, PC_NUM_COMP_SOLANA); - - assert_eq!( - size_of::(), - 48 + u64::BITS as usize - + 3 * size_of::() - + size_of::() - + (PC_NUM_COMP_SOLANA as usize) * size_of::() - ); - assert_eq!(size_of::(), 3312); - assert!(size_of::() <= try_convert::<_, usize>(ZSTD_UPPER_BOUND).unwrap()); - } - assert_eq!(size_of::(), 8); assert_eq!(size_of::(), 16); assert_eq!(size_of::(), 16); diff --git a/scripts/build-bpf.sh b/scripts/build-bpf.sh index 41fc82739..d2276803f 100755 --- a/scripts/build-bpf.sh +++ b/scripts/build-bpf.sh @@ -24,13 +24,6 @@ cargo test --locked # Re-run tests affected by features cargo test --locked --features pythnet -cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort -sha256sum ./target/**/*.so -echo "Checking size of pyth_oracle.so for mainnet" -./scripts/check-size.sh 81760 -mkdir -p target/pyth/solana/ -mv target/deploy/pyth_oracle.so target/pyth/solana/pyth_oracle_solana.so - cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort --features pythnet sha256sum ./target/**/*.so echo "Checking size of pyth_oracle.so for pythnet" From 9f27d4b711e60f1bb709727ca98030edacae101f Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 22 Mar 2024 14:58:47 +0900 Subject: [PATCH 02/13] remove default feature --- program/rust/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 1c2ff744e..2b88bdab9 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -43,7 +43,6 @@ lazy_static = "1.4.0" # to cargo-build-bpf at that point - we manually capture them at # compile-time and pass on to the child process. [features] -default = ["pythnet"] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] library = [] From 3099d2d15010824d6053b936d62c192be2de1d16 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 22 Mar 2024 15:53:35 +0900 Subject: [PATCH 03/13] fix ci --- scripts/build-bpf.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build-bpf.sh b/scripts/build-bpf.sh index d2276803f..08b6c48e4 100755 --- a/scripts/build-bpf.sh +++ b/scripts/build-bpf.sh @@ -19,8 +19,6 @@ set -x # Build both parts of the program (both C and rust) via Cargo cd "${PYTH_DIR}" -cargo test --locked - # Re-run tests affected by features cargo test --locked --features pythnet From 19505efcb5157f8169edb28721b70645ff5af39b Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Mon, 25 Mar 2024 23:31:37 +0900 Subject: [PATCH 04/13] fix readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6accaccaa..efc172f73 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ Finally, in docker extension inside VS Code click right and choose "Attach VS Co Oracle program upgrades are managed by the Pythian Council multisig. The steps to deploy a new version are: -1. Create a release branch from `main`. This should include binary for both the Pythnet oracle programs (`pyth_oracle_pythnet.so`). +1. Create a release branch from `main`. This should include binary for the Pythnet oracle program (`pyth_oracle_pythnet.so`). 2. [Install Solana CLI](https://docs.solana.com/cli/install-solana-cli-tools#use-solanas-install-tool) if not already installed. 3. Set Solana config for the target network, e.g., devnet: `solana config set --url https://api.devnet.solana.com`. 4. Execute `solana program write-buffer pyth_oracle_pythnet.so` to obtain the buffer address. From 78078c279f50478d24356026ba79f5a7476f9be5 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 26 Mar 2024 13:58:42 +0900 Subject: [PATCH 05/13] refactor --- .pre-commit-config.yaml | 45 +++++++++++------------- README.md | 2 +- program/rust/Cargo.toml | 1 - program/rust/build.rs | 11 +++--- program/rust/src/tests/pyth_simulator.rs | 22 ------------ scripts/build-bpf.sh | 4 +-- 6 files changed, 27 insertions(+), 58 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbb678117..0966fd1ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,26 +1,21 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: trailing-whitespace - files : program/ - - id: end-of-file-fixer - files : program/ - - id: check-added-large-files -- repo: local - hooks: - - id: cargo-fmt - name: Cargo Fmt - language: "rust" - entry: cargo +nightly-2023-03-01 fmt - pass_filenames: false - - id: cargo-clippy-solana - name: Cargo Clippy Solana - language: "rust" - entry : cargo +nightly-2023-03-01 clippy --tests --features check -- -D warnings - pass_filenames : false - - id: cargo-clippy-pythnet - name: Cargo Clippy Pythnet - language: "rust" - entry : cargo +nightly-2023-03-01 clippy --tests --features pythnet,check -- -D warnings - pass_filenames : false + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + files: program/ + - id: end-of-file-fixer + files: program/ + - id: check-added-large-files + - repo: local + hooks: + - id: cargo-fmt + name: Cargo Fmt + language: "rust" + entry: cargo +nightly-2023-03-01 fmt + pass_filenames: false + - id: cargo-clippy-pythnet + name: Cargo Clippy Pythnet + language: "rust" + entry: cargo +nightly-2023-03-01 clippy --tests -- -D warnings + pass_filenames: false diff --git a/README.md b/README.md index efc172f73..93d62a696 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ Finally, in docker extension inside VS Code click right and choose "Attach VS Co Oracle program upgrades are managed by the Pythian Council multisig. The steps to deploy a new version are: -1. Create a release branch from `main`. This should include binary for the Pythnet oracle program (`pyth_oracle_pythnet.so`). +1. Create a release branch from `main`. This should include the binary for the Pythnet oracle program (`pyth_oracle_pythnet.so`). 2. [Install Solana CLI](https://docs.solana.com/cli/install-solana-cli-tools#use-solanas-install-tool) if not already installed. 3. Set Solana config for the target network, e.g., devnet: `solana config set --url https://api.devnet.solana.com`. 4. Execute `solana program write-buffer pyth_oracle_pythnet.so` to obtain the buffer address. diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 2b88bdab9..37c045efc 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -46,7 +46,6 @@ lazy_static = "1.4.0" check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] library = [] -pythnet = [] # logic-affecting features start with this one [lib] crate-type = ["cdylib", "lib"] diff --git a/program/rust/build.rs b/program/rust/build.rs index 7d746c0a3..af8209229 100644 --- a/program/rust/build.rs +++ b/program/rust/build.rs @@ -12,7 +12,6 @@ use { fn main() { let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - let has_feat_pythnet = std::env::var("CARGO_FEATURE_PYTHNET").is_ok(); let has_feat_check = std::env::var("CARGO_FEATURE_CHECK").is_ok(); // OUT_DIR is the path cargo provides to a build directory under `target/` specifically for @@ -28,13 +27,11 @@ fn main() { let mut make_extra_flags = vec![]; let mut clang_extra_flags = vec![]; - if has_feat_pythnet { - // Define PC_PYTHNET for the C binary build - make_extra_flags.push("PC_PYTHNET=1"); + // Define PC_PYTHNET for the C binary build + make_extra_flags.push("PC_PYTHNET=1"); - // Define PC_PYTHNET for the bindings build - clang_extra_flags.push("-DPC_PYTHNET=1"); - } + // Define PC_PYTHNET for the bindings build + clang_extra_flags.push("-DPC_PYTHNET=1"); let mut make_targets = vec![]; if target_arch == "bpf" { diff --git a/program/rust/src/tests/pyth_simulator.rs b/program/rust/src/tests/pyth_simulator.rs index 2c10a6ab0..f1f9c80cf 100644 --- a/program/rust/src/tests/pyth_simulator.rs +++ b/program/rust/src/tests/pyth_simulator.rs @@ -75,28 +75,6 @@ lazy_static::lazy_static! { // simulator. lazy_static makes this happen only once per test // run. static ref ORACLE_PROGRAM_BINARY_PATH: PathBuf = { - - // Detect features and pass them onto cargo-build-bpf. - // IMPORTANT: All features of this crate must have gates added to this vector. - let features: Vec<&str> = vec!["pythnet"]; - - let mut cmd = std::process::Command::new("cargo"); - cmd.arg("build-bpf"); - - if !features.is_empty() { - cmd.arg("--features"); - cmd.args(features); - } - - let status = cmd.status().unwrap(); - - if !status.success() { - panic!( - "cargo-build-bpf did not exit with 0 (code {:?})", - status.code() - ); - } - let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); PathBuf::from(target_dir).join("deploy/pyth_oracle.so") diff --git a/scripts/build-bpf.sh b/scripts/build-bpf.sh index 08b6c48e4..f33683065 100755 --- a/scripts/build-bpf.sh +++ b/scripts/build-bpf.sh @@ -20,9 +20,9 @@ set -x cd "${PYTH_DIR}" # Re-run tests affected by features -cargo test --locked --features pythnet +cargo test --locked -cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort --features pythnet +cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort sha256sum ./target/**/*.so echo "Checking size of pyth_oracle.so for pythnet" ./scripts/check-size.sh 88429 From a43a6a099f4b5044e38aba7e32b716861000bc14 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 26 Mar 2024 14:14:37 +0900 Subject: [PATCH 06/13] try fix --- Cargo.lock | 1 - program/rust/Cargo.toml | 8 -------- program/rust/src/tests/pyth_simulator.rs | 15 +++------------ 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ee6e49dc..933b7125a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2514,7 +2514,6 @@ dependencies = [ "byteorder", "csv", "hex 0.3.2", - "lazy_static", "num-derive", "num-traits", "pythnet-sdk", diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 37c045efc..df7537556 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -33,15 +33,7 @@ pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60 serde_json = "1.0" test-generator = "0.3.1" csv = "1.1" -lazy_static = "1.4.0" -# IMPORTANT: Features which affect oracle business logic must be added -# to the `lazy_static` macro call in `pyth_simulator.rs`. -# -# Context: We perform a cargo-build-bpf call as part of running tests -# to obtain a BPF binary for testing. Desired features are not known -# to cargo-build-bpf at that point - we manually capture them at -# compile-time and pass on to the child process. [features] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] diff --git a/program/rust/src/tests/pyth_simulator.rs b/program/rust/src/tests/pyth_simulator.rs index f1f9c80cf..876405039 100644 --- a/program/rust/src/tests/pyth_simulator.rs +++ b/program/rust/src/tests/pyth_simulator.rs @@ -70,17 +70,6 @@ use { }, }; -lazy_static::lazy_static! { - // Build the oracle binary and make it available to the - // simulator. lazy_static makes this happen only once per test - // run. - static ref ORACLE_PROGRAM_BINARY_PATH: PathBuf = { - let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); - - PathBuf::from(target_dir).join("deploy/pyth_oracle.so") - }; -} - /// Simulator for the state of the pyth program on Solana. You can run solana transactions against /// this struct to test how pyth instructions execute in the Solana runtime. pub struct PythSimulator { @@ -113,7 +102,9 @@ struct ProductMetadata { impl PythSimulator { /// Deploys the oracle program as upgradable pub async fn new() -> PythSimulator { - let mut bpf_data = read_file(&*ORACLE_PROGRAM_BINARY_PATH); + let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); + let oracle_program_binary_path = PathBuf::from(target_dir).join("deploy/pyth_oracle.so"); + let mut bpf_data = read_file(&oracle_program_binary_path); let mut program_test = ProgramTest::default(); let program_key = Pubkey::new_unique(); From 5d04587ab70219d38bbfeb52ac03c7d4c23df764 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 26 Mar 2024 17:33:06 +0900 Subject: [PATCH 07/13] address comments --- program/rust/src/tests/pyth_simulator.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/program/rust/src/tests/pyth_simulator.rs b/program/rust/src/tests/pyth_simulator.rs index 876405039..389881b4b 100644 --- a/program/rust/src/tests/pyth_simulator.rs +++ b/program/rust/src/tests/pyth_simulator.rs @@ -102,6 +102,16 @@ struct ProductMetadata { impl PythSimulator { /// Deploys the oracle program as upgradable pub async fn new() -> PythSimulator { + let mut cmd = std::process::Command::new("cargo"); + cmd.arg("build-bpf"); + let status = cmd.status().unwrap(); + + if !status.success() { + panic!( + "cargo-build-bpf did not exit with 0 (code {:?})", + status.code() + ); + } let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); let oracle_program_binary_path = PathBuf::from(target_dir).join("deploy/pyth_oracle.so"); let mut bpf_data = read_file(&oracle_program_binary_path); From d649901769d62ac652a4c5b426841be5c1b0d084 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 26 Mar 2024 18:54:02 +0900 Subject: [PATCH 08/13] fix ci --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0966fd1ea..b57c3a770 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,5 +17,5 @@ repos: - id: cargo-clippy-pythnet name: Cargo Clippy Pythnet language: "rust" - entry: cargo +nightly-2023-03-01 clippy --tests -- -D warnings + entry: cargo +nightly-2023-03-01 clippy --tests --features check -- -D warnings pass_filenames: false From 47115ab971dcf2348cc1c1e3fc6a3dfab5d5d7c7 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 26 Mar 2024 22:32:37 +0900 Subject: [PATCH 09/13] remove PC_PYTHNET flag --- program/c/makefile | 9 +-------- program/c/src/oracle/native/upd_aggregate.c | 4 ---- program/c/src/oracle/oracle.h | 9 +-------- program/c/src/oracle/upd_aggregate.c | 6 +----- program/rust/build.rs | 15 ++------------- 5 files changed, 5 insertions(+), 38 deletions(-) diff --git a/program/c/makefile b/program/c/makefile index 563661b1c..3b791858a 100644 --- a/program/c/makefile +++ b/program/c/makefile @@ -11,14 +11,7 @@ else include $(SOLANA)/sdk/sbf/c/sbf.mk endif -# Propagate the PC_PYTHNET feature by conditionally defining it in a -# features.h header. The makefile included from Solana SDK does not -# have an easy way to pass extra C flags which motivates this approach. -ifdef PC_PYTHNET - FEATURES_H_BODY:="\#pragma once\n\#define PC_PYTHNET 1" -else - FEATURES_H_BODY:="\#pragma once" -endif +FEATURES_H_BODY:="\#pragma once" .PHONY: features.h # Putting this in .PHONY makes sure the header is always regenerated diff --git a/program/c/src/oracle/native/upd_aggregate.c b/program/c/src/oracle/native/upd_aggregate.c index 7c3006858..ef6220d8f 100644 --- a/program/c/src/oracle/native/upd_aggregate.c +++ b/program/c/src/oracle/native/upd_aggregate.c @@ -9,11 +9,7 @@ char heap_start[8192]; #include "../upd_aggregate.h" #include "../features.h" -#ifdef PC_PYTHNET extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#else -extern bool c_upd_aggregate_solana( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#endif return upd_aggregate(ptr, slot, timestamp ); } diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index b4e95b534..89a32fc85 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -27,12 +27,9 @@ extern "C" { #define PC_NUM_COMP_PYTHNET 128 // PC_NUM_COMP - number of price components in use -#ifdef PC_PYTHNET // Not whole PC_NUM_COMP_PYTHNET because of stack issues appearing in upd_aggregate() #define PC_NUM_COMP 64 -#else -#define PC_NUM_COMP PC_NUM_COMP_SOLANA -#endif + #define PC_PROD_ACC_SIZE 512 #define PC_EXP_DECAY -9 @@ -209,7 +206,6 @@ typedef struct pc_price pc_price_comp_t comp_[PC_NUM_COMP];// component prices } pc_price_t; -#ifdef PC_PYTHNET // Replace Solana component size's contribution with Pythnet's #define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (3312 \ @@ -220,9 +216,6 @@ typedef struct pc_price static_assert( sizeof( pc_price_t ) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, "" ); #undef PC_EXPECTED_PRICE_SIZE_PYTHNET -#else -static_assert( sizeof( pc_price_t ) == 3312, "" ); -#endif // This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd. // It is set tighly to the current price account + 96 component prices + 48 bytes for cumulative sums diff --git a/program/c/src/oracle/upd_aggregate.c b/program/c/src/oracle/upd_aggregate.c index 5b2a6af63..0c723a698 100644 --- a/program/c/src/oracle/upd_aggregate.c +++ b/program/c/src/oracle/upd_aggregate.c @@ -6,12 +6,8 @@ #include "upd_aggregate.h" #include "features.h" -// Dynamically deciding the name prevents linking the wrong C binary flavor -#ifdef PC_PYTHNET + extern bool c_upd_aggregate_pythnet( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#else -extern bool c_upd_aggregate_solana( pc_price_t *ptr, uint64_t slot, int64_t timestamp ){ -#endif return upd_aggregate(ptr, slot, timestamp ); } diff --git a/program/rust/build.rs b/program/rust/build.rs index af8209229..b33d8dd7c 100644 --- a/program/rust/build.rs +++ b/program/rust/build.rs @@ -24,15 +24,6 @@ fn main() { eprintln!("OUT_DIR is {}", out_dir); let out_dir = PathBuf::from(out_dir); - let mut make_extra_flags = vec![]; - let mut clang_extra_flags = vec![]; - - // Define PC_PYTHNET for the C binary build - make_extra_flags.push("PC_PYTHNET=1"); - - // Define PC_PYTHNET for the bindings build - clang_extra_flags.push("-DPC_PYTHNET=1"); - let mut make_targets = vec![]; if target_arch == "bpf" { make_targets.push("cpyth-bpf"); @@ -48,7 +39,7 @@ fn main() { if has_feat_check { eprintln!("WARNING: `check` feature active, make build is skipped"); } else { - do_make_build(make_extra_flags, make_targets, &out_dir); + do_make_build(make_targets, &out_dir); // Link against the right library for the architecture if target_arch == "bpf" { @@ -71,7 +62,6 @@ fn main() { // Generate and write bindings let bindings = Builder::default() .clang_arg(format!("-I{:}", get_solana_inc_path().display())) - .clang_args(clang_extra_flags) .header("./src/bindings.h") .rustfmt_bindings(true) .generate() @@ -85,14 +75,13 @@ fn main() { println!("cargo:rerun-if-changed=../"); } -fn do_make_build(extra_flags: Vec<&str>, targets: Vec<&str>, out_dir: &Path) { +fn do_make_build(targets: Vec<&str>, out_dir: &Path) { // We must forward OUT_DIR as an env variable to the make script otherwise it will output // its artifacts to the wrong place. let make_output = std::process::Command::new("make") .env("VERBOSE", "1") .env("OUT_DIR", out_dir.display().to_string()) .current_dir("../c") - .args(extra_flags) .args(targets) .output() .expect("Failed to run make for C oracle program"); From efd6e5292e0fbc4b53c12a0dee0b542b4b1bfed3 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Tue, 26 Mar 2024 22:39:15 +0900 Subject: [PATCH 10/13] refactor --- program/rust/src/processor/upd_price.rs | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 524201239..5e62f241d 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -163,7 +163,7 @@ pub fn upd_price( // Try to update the aggregate #[allow(unused_variables)] - let aggregate_updated = if clock.slot > latest_aggregate_price.pub_slot_ { + if clock.slot > latest_aggregate_price.pub_slot_ { let updated = unsafe { // NOTE: c_upd_aggregate must use a raw pointer to price // data. Solana's `.borrow_*` methods require exclusive @@ -184,26 +184,22 @@ pub fn upd_price( unsafe { c_upd_twap(price_account.try_borrow_mut_data()?.as_mut_ptr(), agg_diff); } + let mut price_data = + load_checked::(price_account, cmd_args.header.version)?; + // We want to send a message every time the aggregate price updates. However, during the migration, + // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag + // ensures that after every aggregate update, the next publisher who provides the accumulator accounts + // will send the message. + price_data.message_sent_ = 0; + price_data.update_price_cumulative()?; } - updated - } else { - false - }; + } // Reload price data as a struct after c_upd_aggregate() borrow is dropped let mut price_data = load_checked::(price_account, cmd_args.header.version)?; // Feature-gated accumulator-specific code, used only on pythnet/pythtest { - // We want to send a message every time the aggregate price updates. However, during the migration, - // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag - // ensures that after every aggregate update, the next publisher who provides the accumulator accounts - // will send the message. - if aggregate_updated { - price_data.message_sent_ = 0; - price_data.update_price_cumulative()?; - } - if let Some(accumulator_accounts) = maybe_accumulator_accounts { if price_data.message_sent_ == 0 { // Check that the oracle PDA is correctly configured for the program we are calling. From 10dce76f03311d3c3faae50f716480a57c0f2e45 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 27 Mar 2024 12:52:42 +0900 Subject: [PATCH 11/13] address comments --- program/rust/src/tests/pyth_simulator.rs | 10 ---------- scripts/build-bpf.sh | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/program/rust/src/tests/pyth_simulator.rs b/program/rust/src/tests/pyth_simulator.rs index 389881b4b..876405039 100644 --- a/program/rust/src/tests/pyth_simulator.rs +++ b/program/rust/src/tests/pyth_simulator.rs @@ -102,16 +102,6 @@ struct ProductMetadata { impl PythSimulator { /// Deploys the oracle program as upgradable pub async fn new() -> PythSimulator { - let mut cmd = std::process::Command::new("cargo"); - cmd.arg("build-bpf"); - let status = cmd.status().unwrap(); - - if !status.success() { - panic!( - "cargo-build-bpf did not exit with 0 (code {:?})", - status.code() - ); - } let target_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target"); let oracle_program_binary_path = PathBuf::from(target_dir).join("deploy/pyth_oracle.so"); let mut bpf_data = read_file(&oracle_program_binary_path); diff --git a/scripts/build-bpf.sh b/scripts/build-bpf.sh index f33683065..fd85e4fa0 100755 --- a/scripts/build-bpf.sh +++ b/scripts/build-bpf.sh @@ -20,7 +20,7 @@ set -x cd "${PYTH_DIR}" # Re-run tests affected by features -cargo test --locked +cargo-test-bpf cargo-build-bpf -- --locked -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort sha256sum ./target/**/*.so From e6cc1816f987e1204fa2684929306f2e64e28b02 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 27 Mar 2024 21:05:32 +0900 Subject: [PATCH 12/13] address comments --- program/c/src/oracle/oracle.h | 39 +++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index 89a32fc85..49209c00a 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -23,7 +23,6 @@ extern "C" { #define PC_MAP_TABLE_SIZE 640 // Total price component slots available -#define PC_NUM_COMP_SOLANA 32 #define PC_NUM_COMP_PYTHNET 128 // PC_NUM_COMP - number of price components in use @@ -208,13 +207,45 @@ typedef struct pc_price // Replace Solana component size's contribution with Pythnet's -#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (3312 \ - - PC_NUM_COMP_SOLANA * sizeof(pc_price_comp_t) \ +/* +The value 240 is derived from the fixed size of the pc_price_t struct excluding the size of the comp_ array. +Here is the breakdown of the sizes (in bytes) for each field within the pc_price_t struct: + +- magic_ (uint32_t): 4 bytes +- ver_ (uint32_t): 4 bytes +- type_ (uint32_t): 4 bytes +- size_ (uint32_t): 4 bytes +- ptype_ (uint32_t): 4 bytes +- expo_ (int32_t): 4 bytes +- num_ (uint32_t): 4 bytes +- num_qt_ (uint32_t): 4 bytes +- last_slot_ (uint64_t): 8 bytes +- valid_slot_ (uint64_t): 8 bytes +- twap_ (pc_ema_t): 24 bytes (3 fields of int64_t each taking 8 bytes) +- twac_ (pc_ema_t): 24 bytes (similar to twap_) +- timestamp_ (int64_t): 8 bytes +- min_pub_ (uint8_t): 1 byte +- message_sent_ (int8_t): 1 byte +- max_latency_ (uint8_t): 1 byte +- drv3_ (int8_t): 1 byte +- drv4_ (int32_t): 4 bytes +- prod_ (pc_pub_key_t): 32 bytes (assuming pc_pub_key_t is a 32-byte array or struct) +- next_ (pc_pub_key_t): 32 bytes (similar to prod_) +- prev_slot_ (uint64_t): 8 bytes +- prev_price_ (int64_t): 8 bytes +- prev_conf_ (uint64_t): 8 bytes +- prev_timestamp_ (int64_t): 8 bytes +- agg_ (pc_price_info_t): 32 bytes + +Adding up all these sizes gives us a total of 240 bytes for the fixed part of the pc_price_t struct. +The size of the comp_ array is variable and depends on PC_NUM_COMP, hence it is calculated separately and added to the base size of 240 bytes. +*/ +#define PC_EXPECTED_PRICE_T_SIZE_PYTHNET (240 \ + PC_NUM_COMP * sizeof(pc_price_comp_t) \ ) static_assert( sizeof( pc_price_t ) == PC_EXPECTED_PRICE_T_SIZE_PYTHNET, "" ); -#undef PC_EXPECTED_PRICE_SIZE_PYTHNET +#undef PC_EXPECTED_PRICE_T_SIZE_PYTHNET // This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd. From ef8fc82e766a70c021694ef43b489ce49864e299 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Wed, 27 Mar 2024 22:11:35 +0900 Subject: [PATCH 13/13] remove comment --- program/c/src/oracle/oracle.h | 1 - 1 file changed, 1 deletion(-) diff --git a/program/c/src/oracle/oracle.h b/program/c/src/oracle/oracle.h index 49209c00a..05c623389 100644 --- a/program/c/src/oracle/oracle.h +++ b/program/c/src/oracle/oracle.h @@ -206,7 +206,6 @@ typedef struct pc_price } pc_price_t; -// Replace Solana component size's contribution with Pythnet's /* The value 240 is derived from the fixed size of the pc_price_t struct excluding the size of the comp_ array. Here is the breakdown of the sizes (in bytes) for each field within the pc_price_t struct: