Merge branch 'main' into same-slot-agg

Author: cctdaniel

.gitignore

@@ -10,4 +10,7 @@ features.h
 
 # IntelliJ / CLion configuration
 .idea
-*.iml
+*.iml
+
+# CMake files
+features.h

Cargo.lock

@@ -13,14 +13,14 @@ First, make sure you have the [solana tool suite](https://docs.solana.com/cli/in
 installed on your machine. (The build depends on some C makefiles that are in the tool suite.)
 Make sure you have installed the same solana version that is being used in [CI](.github/workflows/docker.yaml)
 
-Then, simply run `cargo build` to compile the oracle program as a native binary, or `cargo build-bpf` to build a BPF binary 
+Then, simply run `cargo build` to compile the oracle program as a native binary, or `cargo build-bpf` to build a BPF binary
 that can be uploaded to the blockchain. This step will produce a program binary `target/deploy/pyth_oracle.so`.
 
 ### Testing
 
-Simply run `cargo test`. This command will run several sets of tests:  
+Simply run `cargo test`. This command will run several sets of tests:
 
-- Unit tests of individual functions 
+- Unit tests of individual functions
 - Simulated transaction tests against the BPF binary running on a solana simulator
 - Exhaustive / randomized test batteries for core oracle functionality
 
@@ -30,6 +30,7 @@ The C tests are linked into the rust binary so they run as part of `cargo test`
 You can also run `cargo test-bpf`, which runs the same tests as `cargo test`, though it's slightly slower and the UX is worse.
 
 ### pre-commit hooks
+
 pre-commit is a tool that checks and fixes simple issues (formatting, ...) before each commit. You can install it by following [their website](https://pre-commit.com/). In order to enable checks for this repo run `pre-commit install` from command-line in the root of this repo.
 
 The checks are also performed in the CI to ensure the code follows consistent formatting. Formatting is only currently enforced in the `program/` directory.
@@ -38,7 +39,7 @@ You might also need to install the nightly toolchain to run the formatting by ru
 ## pythd (off-chain client API)
 
 > :warning: pythd is deprecated and has been replaced by [pyth-agent](https://github.com/pyth-network/pyth-agent).
-> This new client is backward compatible with pythd, but more stable and configurable. 
+> This new client is backward compatible with pythd, but more stable and configurable.
 
 `pythd` provides exposes a web API for interacting with the on-chain oracle program.
 
@@ -139,6 +140,7 @@ docker run --name pyth-dev -d \\
 ```
 
 Default user in the image is `pyth` which may not have access to your directories. Assign your user id and group id to it to enable access.
+
 ```
 host@host$ id $USER # Shows user_id, group_id, and group names
 host@host$ docker exec -ti pyth-dev bash
@@ -148,3 +150,17 @@ root@pyth-dev# usermod -u 1002 -g 1004 -s /bin/bash pyth
 ```
 
 Finally, in docker extension inside VS Code click right and choose "Attach VS Code". If you're using a remote host in VS Code make sure to let this connection be open.
+
+## Deployment
+
+Oracle program upgrades are managed by the Pythian Council multisig. The steps to deploy a new version are:
+
+1. Create a release branch from `main`. This should include binaries for both the Solana mainnet and Pythnet oracle programs (`pyth_oracle_solana.so` and `pyth_oracle_pythnet.so`).
+2. [Install Solana CLI](https://docs.solana.com/cli/install-solana-cli-tools#use-solanas-install-tool) if not already installed.
+3. Set Solana config for the target network, e.g., devnet: `solana config set --url https://api.devnet.solana.com`.
+4. Execute `solana program write-buffer <ORACLE_PROGRAM_BINARY_FILENAME>` where `<ORACLE_PROGRAM_BINARY_FILENAME>` can be `pyth_oracle_solana.so` or `pyth_oracle_pythnet.so` to obtain the buffer address.
+5. Run `solana program show <ORACLE_PROGRAM_PUBKEY>` to obtain the authority of the current program.
+6. Use `solana program set-buffer-authority <BUFFER_PUBKEY> --new-buffer-authority <NEW_BUFFER_AUTHORITY>` to assign the upgrade authority from the previous step to the buffer address.
+7. Submit a proposal with [`xc-admin`](https://github.com/pyth-network/pyth-crosschain/tree/main/governance/xc_admin/packages/xc_admin_cli) for program upgrade using the `upgrade-program` command.
+8. Verify the buffer by running `solana program dump <BUFFER> temp_file && shasum -a 256 temp_file && rm temp_file`, comparing the hash with the one from [build-bpf.sh#L35](https://github.com/pyth-network/pyth-client/blob/main/scripts/build-bpf.sh#L35).
+9. Once the proposal secures enough signatures, it will be automatically relayed to the target network, upgrading the program.

README.md

@@ -195,8 +195,9 @@ typedef struct pc_price
   pc_ema_t        twac_;              // time-weighted average conf interval
   int64_t         timestamp_;         // unix timestamp of aggregate price
   uint8_t         min_pub_;           // min publishers for valid price
-  int8_t          drv2_;              // space for future derived values
-  int16_t         drv3_;              // space for future derived values
+  int8_t          message_sent_;      // flag to indicate if the current aggregate price has been sent as a message to the message buffer, 0 if not sent, 1 if sent
+  uint8_t         max_latency_;       // configurable max latency in slots between send and receive
+  int8_t          drv3_;              // space for future derived values
   int32_t         drv4_;              // space for future derived values
   pc_pub_key_t    prod_;              // product id/ref-account
   pc_pub_key_t    next_;              // next price account in list

program/c/src/oracle/oracle.h

@@ -173,10 +173,13 @@ static inline bool upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest
       int64_t slot_diff = ( int64_t )slot - ( int64_t )( iptr->agg_.pub_slot_ );
       int64_t price     = iptr->agg_.price_;
       int64_t conf      = ( int64_t )( iptr->agg_.conf_ );
+      int64_t max_latency = ptr->max_latency_ ? ptr->max_latency_ : PC_MAX_SEND_LATENCY;
       if ( iptr->agg_.status_ == PC_STATUS_TRADING &&
            // No overflow for INT64_MIN+conf or INT64_MAX-conf as 0 < conf < INT64_MAX
            // These checks ensure that price - conf and price + conf do not overflow.
-           (int64_t)0 < conf && (INT64_MIN + conf) <= price && price <= (INT64_MAX-conf) && slot_diff <= PC_MAX_SEND_LATENCY ) {
+           (int64_t)0 < conf && (INT64_MIN + conf) <= price && price <= (INT64_MAX-conf) &&
+           // slot_diff is implicitly >= 0 due to the check in Rust code ensuring publishing_slot is always less than or equal to the current slot.
+           slot_diff <= max_latency ) {
         numv += 1;
         prcs[ nprcs++ ] = price - conf;
         prcs[ nprcs++ ] = price;

program/c/src/oracle/upd_aggregate.h

@@ -1,6 +1,6 @@
 [package]
 name = "pyth-oracle"
-version = "2.25.0"
+version = "2.26.0"
 edition = "2021"
 license = "Apache 2.0"
 publish = false

program/rust/Cargo.toml

@@ -65,7 +65,10 @@ mod price_pythnet {
         /// Minimum valid publisher quotes for a succesful aggregation
         pub min_pub_:           u8,
         pub message_sent_:      u8,
-        pub unused_2_:          i16,
+        /// Configurable max latency in slots between send and receive
+        pub max_latency_:       u8,
+        /// Unused placeholder for alignment
+        pub unused_2_:          i8,
         pub unused_3_:          i32,
         /// Corresponding product account
         pub product_account:    Pubkey,
@@ -116,6 +119,7 @@ mod price_pythnet {
                     self.agg_.price_,
                     self.agg_.conf_,
                     self.agg_.pub_slot_.saturating_sub(self.prev_slot_),
+                    self.max_latency_,
                 ); // pub_slot should always be >= prev_slot, but we protect ourselves against underflow just in case
                 Ok(())
             } else {
@@ -172,11 +176,17 @@ mod price_pythnet {
     }
 
     impl PriceCumulative {
-        pub fn update(&mut self, price: i64, conf: u64, slot_gap: u64) {
+        pub fn update(&mut self, price: i64, conf: u64, slot_gap: u64, max_latency: u8) {
             self.price += i128::from(price) * i128::from(slot_gap);
             self.conf += u128::from(conf) * u128::from(slot_gap);
+            // Use PC_MAX_SEND_LATENCY if max_latency is 0, otherwise use max_latency
+            let latency = if max_latency == 0 {
+                u64::from(PC_MAX_SEND_LATENCY)
+            } else {
+                u64::from(max_latency)
+            };
             // This is expected to saturate at 0 most of the time (while the feed is up).
-            self.num_down_slots += slot_gap.saturating_sub(PC_MAX_SEND_LATENCY.into());
+            self.num_down_slots += slot_gap.saturating_sub(latency);
         }
     }
 }
@@ -225,7 +235,10 @@ mod price_solana {
         /// Whether the current aggregate price has been sent as a message to the message buffer.
         /// 0 = false, 1 = true. (this is a u8 to make the Pod trait happy)
         pub message_sent_:      u8,
-        pub unused_2_:          i16,
+        /// Configurable max latency in slots between send and receive
+        pub max_latency_:       u8,
+        /// Unused placeholder for alignment
+        pub unused_2_:          i8,
         pub unused_3_:          i32,
         /// Corresponding product account
         pub product_account:    Pubkey,

program/rust/src/accounts/price.rs

@@ -99,6 +99,10 @@ pub enum OracleCommand {
     // key[2] permissions account       [writable]
     // key[3] system program            []
     UpdPermissions        = 17,
+    /// Set max latency
+    // account[0] funding account       [signer writable]
+    // account[1] price account         [signer writable]
+    SetMaxLatency         = 18,
 }
 
 #[repr(C)]
@@ -162,3 +166,11 @@ pub struct UpdPermissionsArgs {
     pub data_curation_authority: Pubkey,
     pub security_authority:      Pubkey,
 }
+
+#[repr(C)]
+#[derive(Zeroable, Clone, Copy, Pod)]
+pub struct SetMaxLatencyArgs {
+    pub header:      CommandHeader,
+    pub max_latency: u8,
+    pub unused_:     [u8; 3],
+}

program/rust/src/instruction.rs

@@ -21,6 +21,7 @@ mod del_product;
 mod del_publisher;
 mod init_mapping;
 mod init_price;
+mod set_max_latency;
 mod set_min_pub;
 mod upd_permissions;
 mod upd_price;
@@ -35,6 +36,7 @@ pub use {
     del_publisher::del_publisher,
     init_mapping::init_mapping,
     init_price::init_price,
+    set_max_latency::set_max_latency,
     set_min_pub::set_min_pub,
     upd_permissions::upd_permissions,
     upd_price::{
@@ -76,5 +78,6 @@ pub fn process_instruction(
         DelPrice => del_price(program_id, accounts, instruction_data),
         DelProduct => del_product(program_id, accounts, instruction_data),
         UpdPermissions => upd_permissions(program_id, accounts, instruction_data),
+        SetMaxLatency => set_max_latency(program_id, accounts, instruction_data),
     }
 }

program/rust/src/processor.rs

@@ -16,8 +16,8 @@ use {
         instruction::AddPriceArgs,
         utils::{
             check_exponent_range,
+            check_permissioned_funding_account,
             check_valid_funding_account,
-            check_valid_signable_account_or_permissioned_funding_account,
             pyth_assert,
         },
         OracleError,
@@ -48,26 +48,24 @@ pub fn add_price(
     )?;
 
 
-    let (funding_account, product_account, price_account, permissions_account_option) =
-        match accounts {
-            [x, y, z] => Ok((x, y, z, None)),
-            [x, y, z, p] => Ok((x, y, z, Some(p))),
-            _ => Err(OracleError::InvalidNumberOfAccounts),
-        }?;
+    let (funding_account, product_account, price_account, permissions_account) = match accounts {
+        [x, y, z, p] => Ok((x, y, z, p)),
+        _ => Err(OracleError::InvalidNumberOfAccounts),
+    }?;
 
     check_valid_funding_account(funding_account)?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         product_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         &cmd_args.header,
     )?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         price_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         &cmd_args.header,
     )?;
 

program/rust/src/processor/add_price.rs

@@ -13,8 +13,8 @@ use {
         },
         instruction::CommandHeader,
         utils::{
+            check_permissioned_funding_account,
             check_valid_funding_account,
-            check_valid_signable_account_or_permissioned_funding_account,
             pyth_assert,
             try_convert,
         },
@@ -41,28 +41,27 @@ pub fn add_product(
     accounts: &[AccountInfo],
     instruction_data: &[u8],
 ) -> ProgramResult {
-    let (funding_account, tail_mapping_account, new_product_account, permissions_account_option) =
+    let (funding_account, tail_mapping_account, new_product_account, permissions_account) =
         match accounts {
-            [x, y, z] => Ok((x, y, z, None)),
-            [x, y, z, p] => Ok((x, y, z, Some(p))),
+            [x, y, z, p] => Ok((x, y, z, p)),
             _ => Err(OracleError::InvalidNumberOfAccounts),
         }?;
 
     let hdr = load::<CommandHeader>(instruction_data)?;
 
     check_valid_funding_account(funding_account)?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         tail_mapping_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         hdr,
     )?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         new_product_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         hdr,
     )?;
 

program/rust/src/processor/add_product.rs

@@ -12,8 +12,8 @@ use {
         },
         instruction::AddPublisherArgs,
         utils::{
+            check_permissioned_funding_account,
             check_valid_funding_account,
-            check_valid_signable_account_or_permissioned_funding_account,
             pyth_assert,
             try_convert,
         },
@@ -46,18 +46,17 @@ pub fn add_publisher(
         ProgramError::InvalidArgument,
     )?;
 
-    let (funding_account, price_account, permissions_account_option) = match accounts {
-        [x, y] => Ok((x, y, None)),
-        [x, y, p] => Ok((x, y, Some(p))),
+    let (funding_account, price_account, permissions_account) = match accounts {
+        [x, y, p] => Ok((x, y, p)),
         _ => Err(OracleError::InvalidNumberOfAccounts),
     }?;
 
     check_valid_funding_account(funding_account)?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         price_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         &cmd_args.header,
     )?;
 

program/rust/src/processor/add_publisher.rs

@@ -10,8 +10,8 @@ use {
         },
         instruction::CommandHeader,
         utils::{
+            check_permissioned_funding_account,
             check_valid_funding_account,
-            check_valid_signable_account_or_permissioned_funding_account,
             pyth_assert,
         },
         OracleError,
@@ -37,28 +37,26 @@ pub fn del_price(
     accounts: &[AccountInfo],
     instruction_data: &[u8],
 ) -> ProgramResult {
-    let (funding_account, product_account, price_account, permissions_account_option) =
-        match accounts {
-            [w, x, y] => Ok((w, x, y, None)),
-            [w, x, y, p] => Ok((w, x, y, Some(p))),
-            _ => Err(OracleError::InvalidNumberOfAccounts),
-        }?;
+    let (funding_account, product_account, price_account, permissions_account) = match accounts {
+        [w, x, y, p] => Ok((w, x, y, p)),
+        _ => Err(OracleError::InvalidNumberOfAccounts),
+    }?;
 
     let cmd_args = load::<CommandHeader>(instruction_data)?;
 
     check_valid_funding_account(funding_account)?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         product_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         cmd_args,
     )?;
-    check_valid_signable_account_or_permissioned_funding_account(
+    check_permissioned_funding_account(
         program_id,
         price_account,
         funding_account,
-        permissions_account_option,
+        permissions_account,
         cmd_args,
     )?;