Add feed index for batch publish (#416)

* batch publish wip

* refactor: separate header checks and account data casts

* feat: feed index (wip)

* feat: assign price feed indexes to old accounts

* feat: init feed index when creating prices and init permission account

* refactor: remove transfer to save contract size

* test: add test for price feed indexes

* refactor: address feedback

* chore: fix some outdated account list docs

Author: Riateche

program/rust/build.rs

@@ -91,8 +91,8 @@ fn do_make_build(targets: Vec<&str>, out_dir: &Path) {
             "C oracle make build did not exit with 0 (code
 	({:?}).\n\nstdout:\n{}\n\nstderr:\n{}",
             make_output.status.code(),
-            String::from_utf8(make_output.stdout).unwrap_or("<non-utf8>".to_owned()),
-            String::from_utf8(make_output.stderr).unwrap_or("<non-utf8>".to_owned())
+            String::from_utf8_lossy(&make_output.stdout),
+            String::from_utf8_lossy(&make_output.stderr)
         );
     }
 }

program/rust/src/accounts.rs

@@ -59,6 +59,7 @@ pub use {
         PriceEma,
         PriceInfo,
         PythOracleSerialize,
+        MAX_FEED_INDEX,
     },
     product::{
         update_product_metadata,
@@ -105,6 +106,11 @@ pub trait PythAccount: Pod {
     /// have. `INITIAL_SIZE` <= `minimum_size()`
     const MINIMUM_SIZE: usize = size_of::<Self>();
 
+    /// Size of the account data on creation. Usually this is the same as `MINIMUM_SIZE` but it's
+    /// different for `PermissionAccount` because we've added new fields to it. In this case
+    /// we cannot increase `MINIMUM_SIZE` because that would break reading the account.
+    const NEW_ACCOUNT_SPACE: usize = Self::MINIMUM_SIZE;
+
     /// Given an `AccountInfo`, verify it is sufficiently large and has the correct discriminator.
     fn initialize<'a>(
         account: &'a AccountInfo,
@@ -139,15 +145,15 @@ pub trait PythAccount: Pod {
         seeds: &[&[u8]],
         version: u32,
     ) -> Result<(), ProgramError> {
-        let target_rent = get_rent()?.minimum_balance(Self::MINIMUM_SIZE);
+        let target_rent = get_rent()?.minimum_balance(Self::NEW_ACCOUNT_SPACE);
 
         if account.data_len() == 0 {
             create(
                 funding_account,
                 account,
                 system_program,
                 program_id,
-                Self::MINIMUM_SIZE,
+                Self::NEW_ACCOUNT_SPACE,
                 target_rent,
                 seeds,
             )?;

program/rust/src/accounts/permission.rs

@@ -11,8 +11,15 @@ use {
         Pod,
         Zeroable,
     },
-    solana_program::pubkey::Pubkey,
-    std::mem::size_of,
+    solana_program::{
+        account_info::AccountInfo,
+        program_error::ProgramError,
+        pubkey::Pubkey,
+    },
+    std::{
+        cell::RefMut,
+        mem::size_of,
+    },
 };
 
 /// This account stores the pubkeys that can execute administrative instructions in the Pyth
@@ -50,9 +57,24 @@ impl PermissionAccount {
             _ => false,
         }
     }
+
+    pub fn load_last_feed_index_mut<'a>(
+        account: &'a AccountInfo,
+    ) -> Result<RefMut<'a, u32>, ProgramError> {
+        let start = size_of::<PermissionAccount>();
+        let end = start + size_of::<u32>();
+        assert_eq!(Self::NEW_ACCOUNT_SPACE, end);
+        if account.data_len() < end {
+            return Err(ProgramError::AccountDataTooSmall);
+        }
+        Ok(RefMut::map(account.try_borrow_mut_data()?, |data| {
+            bytemuck::from_bytes_mut(&mut data[start..end])
+        }))
+    }
 }
 
 impl PythAccount for PermissionAccount {
     const ACCOUNT_TYPE: u32 = PC_ACCTYPE_PERMISSIONS;
-    const INITIAL_SIZE: u32 = size_of::<PermissionAccount>() as u32;
+    const NEW_ACCOUNT_SPACE: usize = size_of::<PermissionAccount>() + size_of::<u32>();
+    const INITIAL_SIZE: u32 = Self::NEW_ACCOUNT_SPACE as u32;
 }

program/rust/src/accounts/price.rs

@@ -69,8 +69,9 @@ mod price_pythnet {
         pub max_latency_:       u8,
         /// Various flags
         pub flags:              PriceAccountFlags,
-        /// Unused placeholder for alignment
-        pub unused_3_:          i32,
+        /// Globally unique price feed index used for publishing.
+        /// Limited to 28 bites so that it can be packed together with trading status in a single u32.
+        pub feed_index:         u32,
         /// Corresponding product account
         pub product_account:    Pubkey,
         /// Next price account in the list
@@ -94,6 +95,10 @@ mod price_pythnet {
         pub price_cumulative:   PriceCumulative,
     }
 
+    // Feed index is limited to 28 bites so that it can be packed
+    // together with trading status in a single u32.
+    pub const MAX_FEED_INDEX: u32 = (1 << 28) - 1;
+
     bitflags! {
         #[repr(C)]
         #[derive(Copy, Clone, Pod, Zeroable)]

program/rust/src/error.rs

@@ -52,6 +52,10 @@ pub enum OracleError {
     PermissionViolation            = 619,
     #[error("NeedsSuccesfulAggregation")]
     NeedsSuccesfulAggregation      = 620,
+    #[error("MaxLastFeedIndexReached")]
+    MaxLastFeedIndexReached        = 621,
+    #[error("FeedIndexAlreadyInitialized")]
+    FeedIndexAlreadyInitialized    = 622,
 }
 
 impl From<OracleError> for ProgramError {

program/rust/src/instruction.rs

@@ -39,9 +39,10 @@ pub enum OracleCommand {
     // account[1] product account       [signer writable]
     UpdProduct            = 3,
     /// Add new price account to a product account
-    // account[0] funding account       [signer writable]
-    // account[1] product account       [signer writable]
-    // account[2] new price account     [signer writable]
+    // account[0] funding account        [signer writable]
+    // account[1] product account        [writable]
+    // account[2] new price account      [writable]
+    // account[3] permissions account    [writable]
     AddPrice              = 4,
     /// Add publisher to symbol account
     // account[0] funding account       [signer writable]
@@ -103,6 +104,11 @@ pub enum OracleCommand {
     // account[0] funding account       [signer writable]
     // account[1] price account         [signer writable]
     SetMaxLatency         = 18,
+    /// Init price feed index
+    // account[0] funding account        [signer writable]
+    // account[1] price account          [writable]
+    // account[2] permissions account    [writable]
+    InitPriceFeedIndex    = 19,
 }
 
 #[repr(C)]

program/rust/src/lib.rs

@@ -1,4 +1,3 @@
-#![deny(warnings)]
 // Allow non upper case globals from C
 #![allow(non_upper_case_globals)]
 
@@ -29,6 +28,7 @@ mod log;
 // While we have `pyth-sdk-rs` which exposes a more friendly interface, this is still useful when a
 // downstream user wants to confirm for example that they can compile against the binary interface
 // of this program for their specific solana version.
+pub use crate::error::OracleError;
 #[cfg(feature = "strum")]
 pub use accounts::MessageType;
 #[cfg(feature = "library")]
@@ -45,8 +45,12 @@ pub use accounts::{
     PythAccount,
     PythOracleSerialize,
 };
+#[cfg(feature = "library")]
+pub use {
+    processor::find_publisher_index,
+    utils::get_status_for_conf_price_ratio,
+};
 use {
-    crate::error::OracleError,
     processor::process_instruction,
     solana_program::entrypoint,
 };

program/rust/src/processor.rs

@@ -1,10 +1,21 @@
 use {
     crate::{
+        accounts::{
+            AccountHeader,
+            PermissionAccount,
+            PythAccount,
+            MAX_FEED_INDEX,
+        },
+        deserialize::load_account_as_mut,
         error::OracleError,
         instruction::{
             load_command_header_checked,
             OracleCommand,
         },
+        utils::{
+            pyth_assert,
+            try_convert,
+        },
     },
     solana_program::{
         entrypoint::ProgramResult,
@@ -21,6 +32,7 @@ mod del_product;
 mod del_publisher;
 mod init_mapping;
 mod init_price;
+mod init_price_feed_index;
 mod set_max_latency;
 mod set_min_pub;
 mod upd_permissions;
@@ -47,11 +59,20 @@ pub use {
     upd_price::{
         c_upd_aggregate,
         c_upd_twap,
+        find_publisher_index,
         upd_price,
         upd_price_no_fail_on_error,
     },
     upd_product::upd_product,
 };
+use {
+    init_price_feed_index::init_price_feed_index,
+    solana_program::{
+        program_error::ProgramError,
+        rent::Rent,
+        sysvar::Sysvar,
+    },
+};
 
 /// Dispatch to the right instruction in the oracle.
 pub fn process_instruction(
@@ -84,5 +105,29 @@ pub fn process_instruction(
         DelProduct => del_product(program_id, accounts, instruction_data),
         UpdPermissions => upd_permissions(program_id, accounts, instruction_data),
         SetMaxLatency => set_max_latency(program_id, accounts, instruction_data),
+        InitPriceFeedIndex => init_price_feed_index(program_id, accounts, instruction_data),
+    }
+}
+
+fn reserve_new_price_feed_index(permissions_account: &AccountInfo) -> Result<u32, ProgramError> {
+    if permissions_account.data_len() < PermissionAccount::NEW_ACCOUNT_SPACE {
+        let new_size = PermissionAccount::NEW_ACCOUNT_SPACE;
+        let rent = Rent::get()?;
+        let new_minimum_balance = rent.minimum_balance(new_size);
+        pyth_assert(
+            permissions_account.lamports() >= new_minimum_balance,
+            ProgramError::AccountNotRentExempt,
+        )?;
+
+        permissions_account.realloc(new_size, true)?;
+        let mut header = load_account_as_mut::<AccountHeader>(permissions_account)?;
+        header.size = try_convert(new_size)?;
     }
+    let mut last_feed_index = PermissionAccount::load_last_feed_index_mut(permissions_account)?;
+    *last_feed_index += 1;
+    pyth_assert(
+        *last_feed_index <= MAX_FEED_INDEX,
+        OracleError::MaxLastFeedIndexReached.into(),
+    )?;
+    Ok(*last_feed_index)
 }

program/rust/src/processor/add_price.rs

@@ -1,4 +1,5 @@
 use {
+    super::reserve_new_price_feed_index,
     crate::{
         accounts::{
             PriceAccount,
@@ -18,6 +19,7 @@ use {
             check_exponent_range,
             check_permissioned_funding_account,
             check_valid_funding_account,
+            check_valid_writable_account,
             pyth_assert,
         },
         OracleError,
@@ -31,9 +33,10 @@ use {
 };
 
 /// Add new price account to a product account
-// account[0] funding account       [signer writable]
-// account[1] product account       [signer writable]
-// account[2] new price account     [signer writable]
+// account[0] funding account        [signer writable]
+// account[1] product account        [writable]
+// account[2] new price account      [writable]
+// account[3] permissions account    [writable]
 pub fn add_price(
     program_id: &Pubkey,
     accounts: &[AccountInfo],
@@ -68,6 +71,7 @@ pub fn add_price(
         permissions_account,
         &cmd_args.header,
     )?;
+    check_valid_writable_account(program_id, permissions_account)?;
 
     let mut product_data =
         load_checked::<ProductAccount>(product_account, cmd_args.header.version)?;
@@ -78,6 +82,7 @@ pub fn add_price(
     price_data.product_account = *product_account.key;
     price_data.next_price_account = product_data.first_price_account;
     price_data.min_pub_ = PRICE_ACCOUNT_DEFAULT_MIN_PUB;
+    price_data.feed_index = reserve_new_price_feed_index(permissions_account)?;
     product_data.first_price_account = *price_account.key;
 
     Ok(())

program/rust/src/processor/init_price_feed_index.rs

@@ -0,0 +1,66 @@
+use {
+    super::reserve_new_price_feed_index,
+    crate::{
+        accounts::PriceAccount,
+        deserialize::{
+            load,
+            load_checked,
+        },
+        instruction::CommandHeader,
+        utils::{
+            check_permissioned_funding_account,
+            check_valid_funding_account,
+            check_valid_writable_account,
+            pyth_assert,
+        },
+        OracleError,
+    },
+    solana_program::{
+        account_info::AccountInfo,
+        entrypoint::ProgramResult,
+        program_error::ProgramError,
+        pubkey::Pubkey,
+    },
+    std::mem::size_of,
+};
+
+/// Init price feed index
+// account[0] funding account        [signer writable]
+// account[1] price account          [writable]
+// account[2] permissions account    [writable]
+pub fn init_price_feed_index(
+    program_id: &Pubkey,
+    accounts: &[AccountInfo],
+    instruction_data: &[u8],
+) -> ProgramResult {
+    let cmd = load::<CommandHeader>(instruction_data)?;
+
+    pyth_assert(
+        instruction_data.len() == size_of::<CommandHeader>(),
+        ProgramError::InvalidArgument,
+    )?;
+
+    let (funding_account, price_account, permissions_account) = match accounts {
+        [x, y, p] => Ok((x, y, p)),
+        _ => Err(OracleError::InvalidNumberOfAccounts),
+    }?;
+
+    check_valid_funding_account(funding_account)?;
+    check_permissioned_funding_account(
+        program_id,
+        price_account,
+        funding_account,
+        permissions_account,
+        cmd,
+    )?;
+    check_valid_writable_account(program_id, permissions_account)?;
+
+    let mut price_account_data = load_checked::<PriceAccount>(price_account, cmd.version)?;
+    pyth_assert(
+        price_account_data.feed_index == 0,
+        OracleError::FeedIndexAlreadyInitialized.into(),
+    )?;
+    price_account_data.feed_index = reserve_new_price_feed_index(permissions_account)?;
+
+    Ok(())
+}