blobstore_test.go

// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package blobstore

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"mime/multipart"
	"mime/quotedprintable"
	"net/http"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"testing"

	"golang.org/x/text/encoding/htmlindex"

	"google.golang.org/appengine"
	"google.golang.org/appengine/internal/aetesting"

	pb "google.golang.org/appengine/internal/blobstore"
)

const rbs = readBufferSize

const charsetUTF8 = "utf-8"
const charsetISO2022JP = "iso-2022-jp"
const nonASCIIStr = "Hello, 世界"

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}

func fakeFetchData(req *pb.FetchDataRequest, res *pb.FetchDataResponse) error {
	i0 := int(req.StartIndex)
	i1 := int(req.EndIndex + 1) // Blobstore's end-indices are inclusive; Go's are exclusive.
	bk := req.BlobKey
	if i := strings.Index(bk, "."); i != -1 {
		// Strip everything past the ".".
		bk = bk[:i]
	}
	switch bk {
	case "a14p":
		const s = "abcdefghijklmnop"
		i0 := min(len(s), i0)
		i1 := min(len(s), i1)
		res.Data = []byte(s[i0:i1])
	case "longBlob":
		res.Data = make([]byte, i1-i0)
		for i := range res.Data {
			res.Data[i] = 'A' + uint8(i0/rbs)
			i0++
		}
	}
	return nil
}

// step is one step of a readerTest.
// It consists of a Reader method to call, the method arguments
// (lenp, offset, whence) and the expected results.
type step struct {
	method  string
	lenp    int
	offset  int64
	whence  int
	want    string
	wantErr error
}

var readerTest = []struct {
	blobKey string
	step    []step
}{
	{"noSuchBlobKey", []step{
		{"Read", 8, 0, 0, "", io.EOF},
	}},
	{"a14p.0", []step{
		// Test basic reads.
		{"Read", 1, 0, 0, "a", nil},
		{"Read", 3, 0, 0, "bcd", nil},
		{"Read", 1, 0, 0, "e", nil},
		{"Read", 2, 0, 0, "fg", nil},
		// Test Seek.
		{"Seek", 0, 2, os.SEEK_SET, "2", nil},
		{"Read", 5, 0, 0, "cdefg", nil},
		{"Seek", 0, 2, os.SEEK_CUR, "9", nil},
		{"Read", 1, 0, 0, "j", nil},
		// Test reads up to and past EOF.
		{"Read", 5, 0, 0, "klmno", nil},
		{"Read", 5, 0, 0, "p", nil},
		{"Read", 5, 0, 0, "", io.EOF},
		// Test ReadAt.
		{"ReadAt", 4, 0, 0, "abcd", nil},
		{"ReadAt", 4, 3, 0, "defg", nil},
		{"ReadAt", 4, 12, 0, "mnop", nil},
		{"ReadAt", 4, 13, 0, "nop", io.EOF},
		{"ReadAt", 4, 99, 0, "", io.EOF},
	}},
	{"a14p.1", []step{
		// Test Seek before any reads.
		{"Seek", 0, 2, os.SEEK_SET, "2", nil},
		{"Read", 1, 0, 0, "c", nil},
		// Test that ReadAt doesn't affect the Read offset.
		{"ReadAt", 3, 9, 0, "jkl", nil},
		{"Read", 3, 0, 0, "def", nil},
	}},
	{"a14p.2", []step{
		// Test ReadAt before any reads or seeks.
		{"ReadAt", 2, 14, 0, "op", nil},
	}},
	{"longBlob.0", []step{
		// Test basic read.
		{"Read", 1, 0, 0, "A", nil},
		// Test that Read returns early when the buffer is exhausted.
		{"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil},
		{"Read", 5, 0, 0, "AA", nil},
		{"Read", 3, 0, 0, "BBB", nil},
		// Test that what we just read is still in the buffer.
		{"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil},
		{"Read", 5, 0, 0, "AABBB", nil},
		// Test ReadAt.
		{"ReadAt", 3, rbs - 4, 0, "AAA", nil},
		{"ReadAt", 6, rbs - 4, 0, "AAAABB", nil},
		{"ReadAt", 8, rbs - 4, 0, "AAAABBBB", nil},
		{"ReadAt", 5, rbs - 4, 0, "AAAAB", nil},
		{"ReadAt", 2, rbs - 4, 0, "AA", nil},
		// Test seeking backwards from the Read offset.
		{"Seek", 0, 2*rbs - 8, os.SEEK_SET, strconv.Itoa(2*rbs - 8), nil},
		{"Read", 1, 0, 0, "B", nil},
		{"Read", 1, 0, 0, "B", nil},
		{"Read", 1, 0, 0, "B", nil},
		{"Read", 1, 0, 0, "B", nil},
		{"Read", 8, 0, 0, "BBBBCCCC", nil},
	}},
	{"longBlob.1", []step{
		// Test ReadAt with a slice larger than the buffer size.
		{"LargeReadAt", 2*rbs - 2, 0, 0, strconv.Itoa(2*rbs - 2), nil},
		{"LargeReadAt", 2*rbs - 1, 0, 0, strconv.Itoa(2*rbs - 1), nil},
		{"LargeReadAt", 2*rbs + 0, 0, 0, strconv.Itoa(2*rbs + 0), nil},
		{"LargeReadAt", 2*rbs + 1, 0, 0, strconv.Itoa(2*rbs + 1), nil},
		{"LargeReadAt", 2*rbs + 2, 0, 0, strconv.Itoa(2*rbs + 2), nil},
		{"LargeReadAt", 2*rbs - 2, 1, 0, strconv.Itoa(2*rbs - 2), nil},
		{"LargeReadAt", 2*rbs - 1, 1, 0, strconv.Itoa(2*rbs - 1), nil},
		{"LargeReadAt", 2*rbs + 0, 1, 0, strconv.Itoa(2*rbs + 0), nil},
		{"LargeReadAt", 2*rbs + 1, 1, 0, strconv.Itoa(2*rbs + 1), nil},
		{"LargeReadAt", 2*rbs + 2, 1, 0, strconv.Itoa(2*rbs + 2), nil},
	}},
}

func TestReader(t *testing.T) {
	for _, rt := range readerTest {
		c := aetesting.FakeSingleContext(t, "blobstore", "FetchData", fakeFetchData)
		r := NewReader(c, appengine.BlobKey(rt.blobKey))
		for i, step := range rt.step {
			var (
				got    string
				gotErr error
				n      int
				offset int64
			)
			switch step.method {
			case "LargeReadAt":
				p := make([]byte, step.lenp)
				n, gotErr = r.ReadAt(p, step.offset)
				got = strconv.Itoa(n)
			case "Read":
				p := make([]byte, step.lenp)
				n, gotErr = r.Read(p)
				got = string(p[:n])
			case "ReadAt":
				p := make([]byte, step.lenp)
				n, gotErr = r.ReadAt(p, step.offset)
				got = string(p[:n])
			case "Seek":
				offset, gotErr = r.Seek(step.offset, step.whence)
				got = strconv.FormatInt(offset, 10)
			default:
				t.Fatalf("unknown method: %s", step.method)
			}
			if gotErr != step.wantErr {
				t.Fatalf("%s step %d: got error %v want %v", rt.blobKey, i, gotErr, step.wantErr)
			}
			if got != step.want {
				t.Fatalf("%s step %d: got %q want %q", rt.blobKey, i, got, step.want)
			}
		}
	}
}

// doPlainTextParseUploadTest tests ParseUpload's decoding of non-file form fields.
// It ensures that MIME multipart parts with Content-Type not equal to
// "message/external-body" (i.e. form fields that are not file uploads) are decoded
// correctly according to the value of their Content-Transfer-Encoding header field.
// If charset is not the empty string it will be set in the request's Content-Type
// header field, and if encoding is not the empty string then the Content-Transfer-Encoding
// header field will be set.
func doPlainTextParseUploadTest(t *testing.T, charset string, encoding string,
	rawContent string, encodedContent string) {
	bodyBuf := &bytes.Buffer{}
	w := multipart.NewWriter(bodyBuf)

	fieldName := "foo"
	hdr := textproto.MIMEHeader{}
	hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", fieldName))

	if charset != "" {
		hdr.Set("Content-Type", fmt.Sprintf("text/plain; charset=%q", charset))
	} else {
		hdr.Set("Content-Type", "text/plain")
	}

	if encoding != "" {
		hdr.Set("Content-Transfer-Encoding", encoding)
	}

	pw, err := w.CreatePart(hdr)
	if err != nil {
		t.Fatalf("error creating part: %v", err)
	}
	pw.Write([]byte(encodedContent))

	if err := w.Close(); err != nil {
		t.Fatalf("error closing multipart writer: %v\n", err)
	}

	req, err := http.NewRequest("POST", "/upload", bodyBuf)
	if err != nil {
		t.Fatalf("error creating request: %v", err)
	}

	req.Header.Set("Content-Type", w.FormDataContentType())
	_, other, err := ParseUpload(req)
	if err != nil {
		t.Fatalf("error parsing upload: %v", err)
	}

	if other[fieldName][0] != rawContent {
		t.Errorf("got %q expected %q", other[fieldName][0], rawContent)
	}
}

func TestParseUploadUTF8Base64Encoding(t *testing.T) {
	encoded := base64.StdEncoding.EncodeToString([]byte(nonASCIIStr))
	doPlainTextParseUploadTest(t, charsetUTF8, "base64", nonASCIIStr, encoded)
}

func TestParseUploadUTF8Base64EncodingMultiline(t *testing.T) {
	testStr := "words words words words words words words words words words words words"
	encoded := "d29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29y\r\nZHMgd29yZHMgd29yZHM="
	doPlainTextParseUploadTest(t, charsetUTF8, "base64", testStr, encoded)
}

func TestParseUploadUTF8QuotedPrintableEncoding(t *testing.T) {
	var encoded bytes.Buffer
	writer := quotedprintable.NewWriter(&encoded)
	writer.Write([]byte(nonASCIIStr))
	writer.Close()

	doPlainTextParseUploadTest(t, charsetUTF8, "quoted-printable", nonASCIIStr,
		encoded.String())
}

func TestParseUploadISO2022JPBase64Encoding(t *testing.T) {
	testStr := "こんにちは"
	encoding, err := htmlindex.Get(charsetISO2022JP)
	if err != nil {
		t.Fatalf("error getting encoding: %v", err)
	}

	charsetEncoded, err := encoding.NewEncoder().String(testStr)
	if err != nil {
		t.Fatalf("error encoding string: %v", err)
	}

	base64Encoded := base64.StdEncoding.EncodeToString([]byte(charsetEncoded))
	doPlainTextParseUploadTest(t, charsetISO2022JP, "base64", testStr, base64Encoded)
}

func TestParseUploadNoEncoding(t *testing.T) {
	doPlainTextParseUploadTest(t, "", "", "Hello", "Hello")
}