Re-add TWAP messages, feature-gate price resizing, optimize caching behavior (#378)

* program/rust: Restore TWAP messages, feature-gate resize logic

* Dockerfile, build-bpf.sh: Optimize caching behavior

* .github: Add Solana for pre-commit runs

* move testing-specific cargo-build-bpf call to pyth_simulator.rs

* Update comments about features

* review nits: Dockerfile - remove ls, utils.rs - ungate send_lamports

Author: drozdziak1

.dockerignore

@@ -1,2 +1,3 @@
 build
 target
+program/rust/codegen

Cargo.lock

@@ -1,10 +1,10 @@
-ARG SOLANA_VERSION
-ARG SOLANA_DOCKER_IMAGE_HASH
+ARG SOLANA_VERSION=v1.14.7
+ARG SOLANA_DOCKER_IMAGE_HASH=9130b4a26ec00c4ba99c023da98c06abb0dba2904c990af7770928e0f7dfd2d5
+
 FROM solanalabs/solana:v${SOLANA_VERSION}@sha256:${SOLANA_DOCKER_IMAGE_HASH}
 
 # Redeclare SOLANA_VERSION in the new build stage.
 # Persist in env for docker run & inspect.
-ARG SOLANA_VERSION
 ENV SOLANA_VERSION="${SOLANA_VERSION}"
 
 RUN apt-get update
@@ -46,15 +46,35 @@ WORKDIR /home/pyth
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
   | sh -s -- -y 
 
-COPY --chown=pyth:pyth . pyth-client/
-
-# Build off-chain binaries.
-RUN cd pyth-client && ./scripts/build.sh
+ENV PATH=${PATH}:/home/pyth/.cargo/bin
 
+COPY --chown=pyth:pyth ./scripts/get-bpf-tools.sh pyth-client/scripts/
 RUN ./pyth-client/scripts/get-bpf-tools.sh
+
 # Apply patch to solana
+COPY --chown=pyth:pyth ./scripts/patch-solana.sh ./scripts/solana.patch pyth-client/scripts/
 RUN ./pyth-client/scripts/patch-solana.sh
 
+# Copy C/C++ sources
+COPY --chown=pyth:pyth ./pc pyth-client/pc
+COPY --chown=pyth:pyth ./pctest pyth-client/pctest
+COPY --chown=pyth:pyth ./pcapps pyth-client/pcapps
+COPY --chown=pyth:pyth ./program/c pyth-client/program/c
+COPY --chown=pyth:pyth ./CMakeLists.txt pyth-client
+COPY --chown=pyth:pyth ./scripts/build.sh pyth-client/scripts/
+
+# Build C/C++ offchain binaries.
+RUN cd pyth-client && ./scripts/build.sh
+
+# layer-cache crates.io packages from Cargo.{toml|lock} and rustup toolchain
+COPY --chown=pyth:pyth program/rust/Cargo.toml pyth-client/program/rust/
+COPY --chown=pyth:pyth Cargo.toml Cargo.lock rust-toolchain pyth-client/
+RUN mkdir -p pyth-client/program/rust/src && touch pyth-client/program/rust/src/lib.rs
+RUN cd pyth-client && cargo fetch --locked
+
+# Do final source code copy to overwrite the placeholder lib.rs
+COPY --chown=pyth:pyth ./ pyth-client/
+
 # Build and test the oracle program.
 RUN cd pyth-client && ./scripts/build-bpf.sh .
 

docker/Dockerfile

@@ -1 +1 @@
-bindings.rs
+codegen/*

program/rust/.gitignore

@@ -33,11 +33,19 @@ pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60
 serde_json = "1.0"
 test-generator = "0.3.1"
 csv = "1.1"
+lazy_static = "1.4.0"
 
+# IMPORTANT: Features in this crate must be added to the `lazy_static` macro call in `pyth_simulator.rs`.
+#
+# Context: We perform a cargo-build-bpf call as part of running tests
+# to obtain a BPF binary for testing. Desired features are not known
+# to cargo-build-bpf at that point - we manually capture them at
+# compile-time and pass on to the child process.
 [features]
 debug = []
 library = []
 pythnet = []
+price_v2_resize = []
 
 [lib]
 crate-type = ["cdylib", "lib"]

program/rust/Cargo.toml

@@ -8,11 +8,6 @@ use {
 
 fn main() {
     let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap();
-    // The tests depend on the BPF build (because we load the BPF program into the solana simulator).
-    // Always build the bpf binary before doing anything else.
-    if target_arch != "bpf" {
-        Command::new("cargo").arg("build-bpf").status().unwrap();
-    }
 
     // OUT_DIR is the path cargo provides to a build directory under `target/` specifically for
     // isolated build artifacts. We use this to build the C program and then link against the
@@ -29,7 +24,6 @@ fn main() {
     }
     make_targets.push("test");
 
-
     // We must forward OUT_DIR as an env variable to the make script otherwise it will output
     // its artifacts to the wrong place.
     std::process::Command::new("make")
@@ -49,6 +43,13 @@ fn main() {
     println!("cargo:rustc-link-lib=static=cpyth-test");
     println!("cargo:rustc-link-search={}", out_dir.display());
 
+    std::fs::create_dir("./codegen").unwrap_or_else(|e| {
+        eprintln!(
+            "Could not create codegen directory (may exist which is fine), error: {}",
+            e
+        );
+    });
+
     // Generate and write bindings
     let bindings = Builder::default()
         .clang_arg(format!("-I{:}", get_solana_inc_path().display()))
@@ -57,7 +58,7 @@ fn main() {
         .generate()
         .expect("Unable to generate bindings");
     bindings
-        .write_to_file("./bindings.rs")
+        .write_to_file("./codegen/bindings.rs")
         .expect("Couldn't write bindings!");
 
     // Rerun the build script if either the rust or C code changes

program/rust/build.rs

@@ -146,7 +146,7 @@ pub struct PriceAccountV2 {
     pub price_cumulative:   PriceCumulative,
 }
 
-
+#[allow(dead_code)]
 impl PriceAccountV2 {
     /// This function gets triggered when there's a succesful aggregation and updates the cumulative sums
     pub fn update_price_cumulative(&mut self) -> Result<(), OracleError> {
@@ -162,7 +162,6 @@ impl PriceAccountV2 {
         }
     }
 
-    #[allow(dead_code)]
     pub fn as_twap_message(&self, key: &Pubkey) -> TwapMessage {
         let publish_time = if self.agg_.status_ == PC_STATUS_TRADING {
             self.timestamp_
@@ -215,7 +214,6 @@ impl PriceCumulative {
     }
 }
 
-
 #[repr(C)]
 #[derive(Copy, Clone, Pod, Zeroable)]
 pub struct PriceComponent {

program/rust/src/accounts/price.rs

@@ -4,7 +4,7 @@
 #![allow(dead_code)]
 
 // Bindings.rs is generated by build.rs to include things defined in bindings.h
-include!("../bindings.rs");
+include!("../codegen/bindings.rs");
 
 /// If ci > price / PC_MAX_CI_DIVISOR, set publisher status to unknown.
 /// (e.g., 20 means ci must be < 5% of price)

program/rust/src/c_oracle_header.rs

@@ -27,6 +27,8 @@ mod upd_permissions;
 mod upd_price;
 mod upd_product;
 
+#[cfg(feature = "price_v2_resize")]
+pub use resize_price_account::resize_price_account;
 pub use {
     add_price::add_price,
     add_product::add_product,
@@ -36,7 +38,6 @@ pub use {
     del_publisher::del_publisher,
     init_mapping::init_mapping,
     init_price::init_price,
-    resize_price_account::resize_price_account,
     set_min_pub::set_min_pub,
     upd_permissions::upd_permissions,
     upd_price::{
@@ -71,7 +72,18 @@ pub fn process_instruction(
         UpdTest => Err(OracleError::UnrecognizedInstruction.into()),
         SetMinPub => set_min_pub(program_id, accounts, instruction_data),
         UpdPriceNoFailOnError => upd_price_no_fail_on_error(program_id, accounts, instruction_data),
-        ResizePriceAccount => resize_price_account(program_id, accounts, instruction_data),
+        ResizePriceAccount => {
+            #[cfg(feature = "price_v2_resize")]
+            {
+                resize_price_account(program_id, accounts, instruction_data)
+            }
+
+            #[cfg(not(feature = "price_v2_resize"))]
+            {
+                solana_program::msg!("Oracle built with price_v2_resize disabled. Bailing out!");
+                Err(OracleError::UnrecognizedInstruction.into())
+            }
+        }
         DelPrice => del_price(program_id, accounts, instruction_data),
         DelProduct => del_product(program_id, accounts, instruction_data),
         UpdPermissions => upd_permissions(program_id, accounts, instruction_data),

program/rust/src/processor.rs

@@ -1,3 +1,5 @@
+#![cfg(feature = "price_v2_resize")]
+
 use {
     crate::{
         accounts::{

program/rust/src/processor/resize_price_account.rs

@@ -1,6 +1,8 @@
 #[cfg(feature = "pythnet")]
 use {
     crate::accounts::{
+        PriceAccountV2,
+        PythAccount,
         PythOracleSerialize,
         UPD_PRICE_WRITE_SEED,
     },
@@ -14,9 +16,7 @@ use {
     crate::{
         accounts::{
             PriceAccount,
-            PriceAccountV2,
             PriceInfo,
-            PythAccount,
         },
         deserialize::{
             load,
@@ -160,7 +160,10 @@ pub fn upd_price(
     }
 
     // Try to update the aggregate
+    #[allow(unused_variables)]
     let mut aggregate_updated = false;
+
+    #[allow(unused_assignments)]
     if clock.slot > latest_aggregate_price.pub_slot_ {
         unsafe {
             aggregate_updated = c_upd_aggregate(
@@ -171,20 +174,39 @@ pub fn upd_price(
         }
     }
 
+    // TWAP message will be stored here if:
+    // * price_account is V2
+    // * accumulator accounts were passed
+    #[cfg(feature = "pythnet")]
+    let mut maybe_twap_msg = None;
 
-    let account_len = price_account.try_data_len()?;
-    if account_len >= PriceAccountV2::MINIMUM_SIZE {
-        let mut price_data =
+    // Feature-gated PriceAccountV2-specific processing, used only on pythnet/pythtest. May populate
+    // maybe_twap_msg for later use in accumulator message cross-call.
+    //
+    // NOTE: This is done here specifically in order to keep price
+    // account data available for borrowing as v1 for remaining v1 processing.
+    #[cfg(feature = "pythnet")]
+    if price_account.try_data_len()? >= PriceAccountV2::MINIMUM_SIZE {
+        let mut price_data_v2 =
             load_checked::<PriceAccountV2>(price_account, cmd_args.header.version)?;
 
         if aggregate_updated {
-            price_data.update_price_cumulative()?;
+            // Update PriceCumulative on aggregation. This is done
+            // here, because PriceCumulative must update
+            // regardless of accumulator accounts being passed.
+            price_data_v2.update_price_cumulative()?;
+        }
+
+        // Populate the TWAP message if accumulator will be used
+        if maybe_accumulator_accounts.is_some() {
+            maybe_twap_msg = Some(price_data_v2.as_twap_message(price_account.key));
         }
     }
 
+    // Reload as V1 for remaining V1-compatible processing
     let mut price_data = load_checked::<PriceAccount>(price_account, cmd_args.header.version)?;
 
-
+    // Feature-gated accumulator-specific code, used only on pythnet/pythtest
     #[cfg(feature = "pythnet")]
     {
         // We want to send a message every time the aggregate price updates. However, during the migration,
@@ -194,6 +216,7 @@ pub fn upd_price(
         if aggregate_updated {
             price_data.message_sent_ = 0;
         }
+
         if let Some(accumulator_accounts) = maybe_accumulator_accounts {
             if price_data.message_sent_ == 0 {
                 // Check that the oracle PDA is correctly configured for the program we are calling.
@@ -225,10 +248,16 @@ pub fn upd_price(
                         is_writable: true,
                     },
                 ];
-                let message = vec![price_data
+
+                let mut message = vec![price_data
                     .as_price_feed_message(price_account.key)
                     .to_bytes()];
 
+                // Append a TWAP message if available
+                if let Some(twap_msg) = maybe_twap_msg {
+                    message.push(twap_msg.to_bytes());
+                }
+
                 // anchor discriminator for "global:put_all"
                 let discriminator: [u8; 8] = [212, 225, 193, 91, 151, 238, 20, 93];
                 let create_inputs_ix = Instruction::new_with_borsh(