Implement the new retrieve endpoint and start on tests

Author: burke

Gemfile

@@ -6,3 +6,4 @@ gem('faraday')
 gem('rbnacl')
 gem('mysql2')
 gem('grpc-tools')
+gem('rubyzip')

Gemfile.lock

@@ -14,6 +14,7 @@ GEM
     mysql2 (0.5.3)
     rbnacl (7.1.1)
       ffi
+    rubyzip (2.3.0)
 
 PLATFORMS
   ruby
@@ -26,6 +27,7 @@ DEPENDENCIES
   mocha
   mysql2
   rbnacl
+  rubyzip
 
 BUNDLED WITH
    2.1.4

cmd/generate-ecdsa-private-key/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/hex"
+	"fmt"
+)
+
+func main() {
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+	data, err := x509.MarshalECPrivateKey(privateKey)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(hex.EncodeToString(data))
+}

dev.yml

@@ -23,6 +23,7 @@ up:
 env:
   KEY_CLAIM_TOKEN: test=ON
   RETRIEVE_HMAC_KEY: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  ECDSA_KEY: 30770201010420a6885a310b694b7bb4ba985459de1e79446dddcd1247c62ece925402b362a110a00a06082a8648ce3d030107a1440342000403eb64f714c4b4ed394331c26c31b7ce7156d00fb28982ad2679a87eaa1a3869802fbeb1d7ee28002762921929c3f7603672d535fcac3d24d57afbb4e2d97f5a
   DATABASE_URL: "root@tcp(covidshield.railgun)/covidshield"
   DB_HOST: covidshield.railgun
   DB_USER: root

examples/retrieval/app.rb

@@ -6,7 +6,7 @@
 require_relative('../../test/lib/protocol/covidshield_pb')
 
 class Database
-  Fetch = Struct.new(:date, :hour)
+  Fetch = Struct.new(:period)
 
   def initialize
     @fetches = []
@@ -17,21 +17,13 @@ def drop_old_data
     @fetches.reject! { |fetch| fetch.date < min_date }
   end
 
-  def has_all_hours?(date)
-    (0..23).all? { |hour| has_hour?(date, hour) }
+  def fetched?(period)
+    @fetches.include?(Fetch.new(period))
   end
 
-  def has_hour?(date, hour)
-    @fetches.include?(Fetch.new(date, hour))
-  end
-
-  def mark_hours_fetched(date)
-    (0..23).each { |hour| mark_hour_fetched(date, hour) }
-  end
-
-  def mark_hour_fetched(date, hour)
-    return if @fetches.include?(Fetch.new(date, hour))
-    @fetches << Fetch.new(date, hour)
+  def mark_fetched(period)
+    return if @fetches.include?(Fetch.new(period))
+    @fetches << Fetch.new(period)
   end
 
   private
@@ -89,15 +81,10 @@ def fetch_new_keys
 
     @database.drop_old_data
 
-    (1..14).to_a.reverse.each do |days_ago| # [1, 14] => 14,13,...,2,1
-      date = today_utc.prev_day(days_ago)
-      fetch_date(date) unless @database.has_all_hours?(date)
-    end
-
-    # ... is exclusive range -- [0, hour)
-    (0...current_hour_number_within_utc_day).each do |hour|
-      date = today_utc
-      fetch_hour(date, hour) unless @database.has_hour?(date, hour)
+    curr = current_period
+    168.times do |n|
+      period = curr - (2 * (n + 1))
+      fetch_period(period) unless @database.fetched?(period)
     end
   end
 
@@ -108,40 +95,25 @@ def fetch_exposure_config(region)
     raise("failed") unless resp['Content-Type'] == 'application/json'
   end
 
-  def fetch_date(date)
-    puts "Fetching date: #{date}"
-    resp = Faraday.get(date_url(date))
+  def fetch_period(period)
+    puts "Fetching period: #{period}"
+    resp = Faraday.get(period_url(period))
     raise("failed") unless resp.status == 200
-    raise("failed") unless resp['Content-Type'] == 'application/x-protobuf; delimited=true'
+    raise("failed") unless resp['Content-Type'] == 'application/zip'
     db_transaction do
-      keys = parse_and_save_keys_from(resp)
-      puts("retrieved #{keys.size} keys")
-      @database.mark_hours_fetched(date)
+      keys = send_to_framework(resp)
+      puts("retrieved pack")
+      @database.mark_fetched(period)
     end
   end
 
-  def fetch_hour(date, hour)
-    puts "Fetching hour: #{date} // #{hour}"
-    resp = Faraday.get(hour_url(date, hour))
-    raise("failed") unless resp.status == 200
-    raise("failed") unless resp['Content-Type'] == 'application/x-protobuf; delimited=true'
-    db_transaction do
-      keys = parse_and_save_keys_from(resp)
-      puts("retrieved #{keys.size} keys")
-      @database.mark_hour_fetched(date, hour)
-    end
+  def current_period
+    (Time.now.to_i / 3600 / 2) * 2
   end
 
-  BIG_ENDIAN_UINT32 = 'N'
-
-  def parse_and_save_keys_from(resp)
-    buf = resp.body.each_byte.to_a
-    files = []
-    until buf.empty?
-      len = buf.shift(4).map(&:chr).join.unpack(BIG_ENDIAN_UINT32).first
-      files << Covidshield::File.decode(buf.shift(len).map(&:chr).join)
-    end
-    files.flat_map(&:key)
+  def send_to_framework(resp)
+    # See retrieve_test.rb for a ruby example of how to load this format, but probably,
+    # you'll just be feeding the response body to the EN framework.
   end
 
   def db_transaction
@@ -152,23 +124,11 @@ def exposure_configuration_url(region)
     "#{KEY_RETRIEVAL_URL}/exposure-configuration/#{region}.json"
   end
 
-  def date_url(date)
-    message = "#{date.iso8601}:#{format("%02d", hour_number)}"
-    key = [ENV.fetch("RETRIEVE_HMAC_KEY")].pack("H*")
-    hmac = OpenSSL::HMAC.hexdigest("SHA256", key, message)
-    "#{KEY_RETRIEVAL_URL}/retrieve-day/#{date.iso8601}/#{hmac}"
-  end
-
-  def hour_url(date, hour)
-    hour = format("%02d", hour)
-    message = "#{date.iso8601}:#{hour}:#{format("%02d", hour_number)}"
+  def period_url(period)
+    message = "#{period}:#{hour_number}"
     key = [ENV.fetch("RETRIEVE_HMAC_KEY")].pack("H*")
     hmac = OpenSSL::HMAC.hexdigest("SHA256", key, message)
-    "#{KEY_RETRIEVAL_URL}/retrieve-hour/#{date.iso8601}/#{hour}/#{hmac}"
-  end
-
-  def current_hour_number_within_utc_day
-    (Time.now.to_i % 86400) / 3600
+    "#{KEY_RETRIEVAL_URL}/retrieve/#{period}/#{hmac}"
   end
 
   def hour_number(at = Time.now)

pkg/app/app.go

@@ -58,16 +58,11 @@ func (a *AppBuilder) WithRetrieval() *AppBuilder {
 
 	a.defaultServerPort = defaultRetrievalServerPort
 
-	key, err := retrieval.DummyKey()
-	if err != nil {
-		panic(err)
-	}
-	signer := retrieval.NewSigner(key)
+	a.components = append(a.components, newExpirationWorker(a.database))
 
 	a.servlets = append(a.servlets, server.NewConfigServlet())
-	a.servlets = append(a.servlets, server.NewRetrieveServlet(a.database, retrieval.NewAuthenticator(), signer))
+	a.servlets = append(a.servlets, server.NewRetrieveServlet(a.database, retrieval.NewAuthenticator(), retrieval.NewSigner()))
 
-	a.components = append(a.components, newExpirationWorker(a.database))
 	return a
 }
 

pkg/retrieval/authenticator.go

@@ -52,13 +52,13 @@ func (a *authenticator) Authenticate(requestedHour string, auth string) bool {
 
 	currentHour := int(timemath.HourNumber(time.Now()))
 
-	if validMAC([]byte(requestedHour+strconv.Itoa(currentHour)), dst, a.hmacKey) {
+	if validMAC([]byte(requestedHour+":"+strconv.Itoa(currentHour)), dst, a.hmacKey) {
 		return true
 	}
-	if validMAC([]byte(requestedHour+strconv.Itoa(currentHour-1)), dst, a.hmacKey) {
+	if validMAC([]byte(requestedHour+":"+strconv.Itoa(currentHour-1)), dst, a.hmacKey) {
 		return true
 	}
-	if validMAC([]byte(requestedHour+strconv.Itoa(currentHour+1)), dst, a.hmacKey) {
+	if validMAC([]byte(requestedHour+":"+strconv.Itoa(currentHour+1)), dst, a.hmacKey) {
 		return true
 	}
 	return false

pkg/retrieval/retrieval.go

@@ -104,16 +104,24 @@ func SerializeTo(
 	if err != nil {
 		return err
 	}
-	if _, err := f.Write(exportBinData); err != nil {
+	n, err := f.Write(exportBinData)
+	if err != nil {
 		return err
 	}
+	if n != len(exportBinData) {
+		panic("len")
+	}
 	f, err = zipw.Create("export.sig")
 	if err != nil {
 		return err
 	}
-	if _, err := f.Write(exportSigData); err != nil {
+	n, err = f.Write(exportSigData)
+	if err != nil {
 		return err
 	}
+	if n != len(exportSigData) {
+		panic("len")
+	}
 
 	return zipw.Close()
 }

pkg/retrieval/signer.go

@@ -5,6 +5,9 @@ import (
 	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"os"
 )
 
 type Signer interface {
@@ -15,8 +18,22 @@ type signer struct {
 	privateKey *ecdsa.PrivateKey
 }
 
-func NewSigner(key *ecdsa.PrivateKey) Signer {
-	return &signer{privateKey: key}
+func NewSigner() Signer {
+	ecdsaKeyHex := os.Getenv("ECDSA_KEY")
+	if ecdsaKeyHex == "" {
+		panic("no ECDSA_KEY")
+	}
+	ecdsaKey, err := hex.DecodeString(ecdsaKeyHex)
+	if err != nil {
+		panic(err)
+	}
+
+	priv, err := x509.ParseECPrivateKey(ecdsaKey)
+	if err != nil {
+		panic(err)
+	}
+
+	return &signer{privateKey: priv}
 }
 
 func (s *signer) Sign(data []byte) ([]byte, error) {
@@ -28,7 +45,3 @@ func (s *signer) Sign(data []byte) ([]byte, error) {
 	signatureX962 := elliptic.Marshal(elliptic.P256(), a, b)
 	return signatureX962, nil
 }
-
-func DummyKey() (*ecdsa.PrivateKey, error) {
-	return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-}