incorrect assumption in mapstructure library causing panic

[jxsl13]

mapstructure package assumes that when reflect.Type.Kind() == reflect.String is true that then interface{}.(string) will always succeed.

Below two tests, the first one panics due to the above assumption, the second one with a fix attempt that does not panic but still needs some improvements from someone who knows more about reflections.

The problem is that we pass a value type to koanf but the pointer type implements the TextMarshaler interface which we want to use before falling back to simply setting the internal string value.

The TODO show where I'm currently struggling...

package main_test

import (
	"encoding"
	"fmt"
	"reflect"
	"strings"
	"testing"

	"github.com/knadh/koanf"
	"github.com/knadh/koanf/providers/structs"
	"github.com/mitchellh/mapstructure"
	"github.com/stretchr/testify/assert"
)

type LogFormat string

func (c *LogFormat) UnmarshalText(data []byte) error {
       //overcomplicated custom internal string representation
        // in order to have a different internal representation from an external string representation
	switch strings.ToLower(string(data)) {
	case "", "json":
		*c = "json_custom"
	case "text":
		*c = "text_custom"
	default:
		return fmt.Errorf("invalid log format: %s", string(data))
	}
	return nil
}

func (c *LogFormat) MarshalText() ([]byte, error) {
	//overcomplicated custom internal string representation
        // in order to have a different internal representation from an external string representation
	switch *c {
	case "", "json_custom":
		return []byte("json"), nil
	case "text_custom":
		return []byte("text"), nil
	}
	return nil, fmt.Errorf("invalid internal string representation: %q", *c)
}

func TestTextUnmarshalStringBroken(t *testing.T) {
	defer func() {
		assert.Nil(t, recover())
	}()

	type targetStruct struct {
		LogFormat LogFormat // default should map to json
	}

	target := &targetStruct{"text_custom"}
	before := target.LogFormat

	k := koanf.New(".")
	k.Load(structs.Provider(target, "koanf"), nil)

	// default values explicitly set at top level in order to see the difference
	err := k.UnmarshalWithConf("",
		&target,
		koanf.UnmarshalConf{
			FlatPaths: true,
			DecoderConfig: &mapstructure.DecoderConfig{
				DecodeHook: mapstructure.ComposeDecodeHookFunc(
					mapstructure.StringToTimeDurationHookFunc(),
					mapstructure.StringToSliceHookFunc(","),
					mapstructure.TextUnmarshallerHookFunc()),
				Metadata:         nil,
				Result:           &target,
				WeaklyTypedInput: true,
			}})
	assert.NoError(t, err)
	assert.Equal(t, before, target.LogFormat)
}

func TestTextUnmarshalStringFixed(t *testing.T) {
	defer func() {
		assert.Nil(t, recover())
	}()

	type targetStruct struct {
		LogFormat LogFormat
	}

	target := &targetStruct{"text_custom"}
	before := target.LogFormat

	var b any = before

	_, ok := (b).(encoding.TextMarshaler)
	assert.True(t, ok)

	k := koanf.New(".")
	k.Load(structs.Provider(target, "koanf"), nil)

	// default values with a custom TextUnmarshalerHookFunc implementation
	err := k.UnmarshalWithConf("",
		&target,
		koanf.UnmarshalConf{
			FlatPaths: true,
			DecoderConfig: &mapstructure.DecoderConfig{
				DecodeHook: mapstructure.ComposeDecodeHookFunc(
					mapstructure.StringToTimeDurationHookFunc(),
					mapstructure.StringToSliceHookFunc(","),
					CustomTextUnmarshalHookFunc()), // our custom implementation
				Metadata:         nil,
				Result:           &target,
				WeaklyTypedInput: true,
			}})
	assert.NoError(t, err)
	assert.Equal(t, before, target.LogFormat)
}

func CustomTextUnmarshalHookFunc() mapstructure.DecodeHookFuncType {
	return func(
		f reflect.Type,
		t reflect.Type,
		data interface{}) (interface{}, error) {
		if f.Kind() != reflect.String {
			return data, nil
		}
		result := reflect.New(t).Interface()
		unmarshaller, ok := result.(encoding.TextUnmarshaler)
		if !ok {
			return data, nil
		}

		// default text representaion is the actual value of the `from` string
		var (
			dataVal = reflect.ValueOf(data)
			text    = []byte(dataVal.String())
		)
		if f.Kind() == t.Kind() {
			// source and target are of underlying type string
			var err error
			ptrVal := reflect.New(reflect.PointerTo(dataVal.Type()))
			ptrVal.SetPointer(dataVal.UnsafePointer())

			// TODO: we need to assert that both, the value type and the pointer type
			// do (not) implement the TextMarshaler interface before proceeding and simmply
			// using the the string value of the string type.
			// it might be the case that the internal string representation differs from
			// the (un)marshalled string.

			for _, v := range []reflect.Value{dataVal, ptrVal} {
				if marshaller, ok := v.Interface().(encoding.TextMarshaler); ok {
					text, err = marshaller.MarshalText()
					if err != nil {
						return nil, err
					}
				}
			}
		}

		if err := unmarshaller.UnmarshalText(text); err != nil {
			return nil, err
		}
		return result, nil
	}
}

func TestReflectPointerValueFromValueValue(t *testing.T) {
	defer func() {
		assert.Nil(t, recover())
	}()

	before := LogFormat("text_custom")
	var (
		b      any = before
		bptr   any = &before
		target     = reflect.ValueOf(bptr)
	)

	_, ok := (b).(encoding.TextMarshaler)
	if ok {
		panic("value type should not implement unmarshaler")
	}

	_, ok = (bptr).(encoding.TextMarshaler)
	if !ok {
		panic("pointer type should implement unmarshaler")
	}

	bVal := reflect.ValueOf(b)

	bptrVal := reflect.New(reflect.PointerTo(bVal.Type()))
	// TODO: do something so that the pointer value of bptrVal becomes the pointer to
	// b which is equivalent to the pointer inside of target



	if !target.Equal(bptrVal) {
		panic("btprValue is not the same as the target pointer value")
	}

	fmt.Println("successfully converted value type to pointer type")
}
```
 

[knadh]

Sorry, haven't had a chance to study this. Were you able to figure this out?

[jxsl13]

currently no time due to private reasons, will come back to this maybe next week.

[jxsl13]

the solution is to make the second test pass by updating the CustomTextUnmarshalHookFunc().
This is an extended form of the issue at hand where a
type Foo string not only is a string under the hood (for which the type assertion .(string) does not work) but that custom string type additionally implements both, the TextMarshaler and TextUnmarshaler interfaces which allow the internal string representation (deserialized) to differ from the external string representation (serialized).
The third test is a test to somehow deduce that the value type as well as the pointer type do not implement the TextMarshaler interface.
The third test is a little bit of a playground for the implementation that I need in CustomTextUnmarshalHookFunc() at TODO
A potential direction is to copy the values and have both pointer type and value type (pointer/value receiver) inside of an empty interface (.Interface()) for which you may then try to type assert TextMarshaler.