Store documents in a memdb-backed table (hashicorp#771)

* internal/document: Decouple document types

* internal/state: Decouple document state into memdb

* internal/filesystem: reduce impl to simple OS FS + interface for DocumentStore

Previously filesystem package had two major use cases, to offer a unified io/fs.FS interface for e.g. parsing *.tf or *.tfvars, which was implemented mostly via external library (spf13/afero). Secondly it also provided a direct full access to the "in-memory layer" of the filesystem for RPC handlers (e.g. didOpen, didChange, didClose, ...).

These use cases rarely overlap throughout the codebase and so this lead to unnecessary imports of the `filesystem` package in places where we only needed either the OS-level FS or in-mem FS, but almost never both.

This decoupling allows us to import `filesystem` or `state.DocumentStore` separately.

Also, as we no longer need the in-mem backend of afero, it makes more sense to just reimplement the small part of the 3rd party library instead.

* internal/hcl: Update references

* internal/lsp: Update references

* internal/source: Update references

* internal/terraform: update references

* internal/*: Update RPC handlers + custom cmds

* replace last occurence of afero with osFs

* internal/document: add tests and comments to Handle+DirHandle

* internal/uri: refactor & add comments

* internal/uri: introduce MustParseURI

* internal/uri: account for VSCode's over-escaping of colon

* internal/document: clean up handle logic & normalize URIs before saving

* internal/uri: account for drive-letter normalization in VSCode

Author: radeksimko

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
-	github.com/spf13/afero v1.8.1
+	github.com/spf13/afero v1.8.1 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/vektra/mockery/v2 v2.10.0
 	github.com/zclconf/go-cty v1.10.0

internal/cmd/completion_command.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/hashicorp/hcl-lang/lang"
 	"github.com/hashicorp/terraform-ls/internal/decoder"
+	"github.com/hashicorp/terraform-ls/internal/document"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
 	"github.com/hashicorp/terraform-ls/internal/logging"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
@@ -63,8 +64,6 @@ func (c *CompletionCommand) Run(args []string) int {
 		return 1
 	}
 
-	fh := ilsp.FileHandlerFromPath(path)
-
 	parts := strings.Split(c.atPos, ":")
 	if len(parts) != 2 {
 		c.Ui.Error(fmt.Sprintf("Error parsing at-pos argument: %q (expected line:col format)", c.atPos))
@@ -84,56 +83,57 @@ func (c *CompletionCommand) Run(args []string) int {
 
 	logger := logging.NewLogger(os.Stderr)
 
-	fs := filesystem.NewFilesystem()
-	fs.SetLogger(logger)
-	fs.CreateAndOpenDocument(fh, "terraform", content)
-
-	doc, err := fs.GetDocument(fh)
+	ss, err := state.NewStateStore()
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1
 	}
 
-	fPos, err := ilsp.FilePositionFromDocumentPosition(lsp.TextDocumentPositionParams{
-		TextDocument: lsp.TextDocumentIdentifier{
-			URI: fh.DocumentURI(),
-		},
-		Position: lspPos,
-	}, doc)
+	dh := document.HandleFromPath(path)
+	err = ss.DocumentStore.OpenDocument(dh, "terraform", 0, content)
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1
 	}
 
-	ctx := context.Background()
-	ss, err := state.NewStateStore()
+	fs := filesystem.NewFilesystem(ss.DocumentStore)
+	fs.SetLogger(logger)
+
+	doc, err := ss.DocumentStore.GetDocument(dh)
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1
 	}
-	modMgr := module.NewSyncModuleManager(ctx, fs, ss.Modules, ss.ProviderSchemas)
 
-	_, err = modMgr.AddModule(fh.Dir())
+	pos, err := ilsp.HCLPositionFromLspPosition(lspPos, doc)
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1
 	}
 
-	pos := fPos.Position()
+	ctx := context.Background()
+
+	modMgr := module.NewSyncModuleManager(ctx, fs, ss.DocumentStore, ss.Modules, ss.ProviderSchemas)
+
+	_, err = modMgr.AddModule(dh.Dir.Path())
+	if err != nil {
+		c.Ui.Error(err.Error())
+		return 1
+	}
 
 	d, err := decoder.NewDecoder(ctx, &decoder.PathReader{
 		ModuleReader: ss.Modules,
 		SchemaReader: ss.ProviderSchemas,
 	}).Path(lang.Path{
-		Path:       doc.Dir(),
-		LanguageID: doc.LanguageID(),
+		Path:       doc.Dir.Path(),
+		LanguageID: doc.LanguageID,
 	})
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1
 	}
 
-	candidates, err := d.CandidatesAtPos(doc.Filename(), pos)
+	candidates, err := d.CandidatesAtPos(doc.Filename, pos)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("failed to find candidates: %s", err.Error()))
 		return 1

internal/cmd/inspect_module_command.go

@@ -84,17 +84,19 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
 		return fmt.Errorf("expected %s to be a directory", rootPath)
 	}
 
-	fs := filesystem.NewFilesystem()
-
-	ctx := context.Background()
 	ss, err := state.NewStateStore()
 	if err != nil {
 		return err
 	}
-	modMgr := module.NewSyncModuleManager(ctx, fs, ss.Modules, ss.ProviderSchemas)
+
+	fs := filesystem.NewFilesystem(ss.DocumentStore)
+
+	ctx := context.Background()
+
+	modMgr := module.NewSyncModuleManager(ctx, fs, ss.DocumentStore, ss.Modules, ss.ProviderSchemas)
 	modMgr.SetLogger(c.logger)
 
-	walker := module.SyncWalker(fs, modMgr)
+	walker := module.SyncWalker(fs, ss.DocumentStore, modMgr)
 	walker.SetLogger(c.logger)
 
 	ctx, cancel := ictx.WithSignalCancel(context.Background(),

internal/context/context.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"time"
 
-	"github.com/hashicorp/terraform-ls/internal/filesystem"
 	"github.com/hashicorp/terraform-ls/internal/langserver/diagnostics"
 	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
 	"github.com/hashicorp/terraform-ls/internal/settings"
@@ -20,7 +19,6 @@ func (k *contextKey) String() string {
 }
 
 var (
-	ctxDs                   = &contextKey{"document storage"}
 	ctxTfExecPath           = &contextKey{"terraform executable path"}
 	ctxTfExecLogPath        = &contextKey{"terraform executor log path"}
 	ctxTfExecTimeout        = &contextKey{"terraform execution timeout"}
@@ -40,19 +38,6 @@ func missingContextErr(ctxKey *contextKey) *MissingContextErr {
 	return &MissingContextErr{ctxKey}
 }
 
-func WithDocumentStorage(ctx context.Context, fs filesystem.DocumentStorage) context.Context {
-	return context.WithValue(ctx, ctxDs, fs)
-}
-
-func DocumentStorage(ctx context.Context) (filesystem.DocumentStorage, error) {
-	fs, ok := ctx.Value(ctxDs).(filesystem.DocumentStorage)
-	if !ok {
-		return nil, missingContextErr(ctxDs)
-	}
-
-	return fs, nil
-}
-
 func WithTerraformExecLogPath(ctx context.Context, path string) context.Context {
 	return context.WithValue(ctx, ctxTfExecLogPath, path)
 }

internal/document/change.go

@@ -0,0 +1,83 @@
+package document
+
+import (
+	"bytes"
+
+	"github.com/hashicorp/terraform-ls/internal/source"
+)
+
+type Change interface {
+	Text() string
+	Range() *Range
+}
+
+type Changes []Change
+
+func ApplyChanges(original []byte, changes Changes) ([]byte, error) {
+	if len(changes) == 0 {
+		return original, nil
+	}
+
+	var buf bytes.Buffer
+	_, err := buf.Write(original)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, ch := range changes {
+		err := applyDocumentChange(&buf, ch)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func applyDocumentChange(buf *bytes.Buffer, change Change) error {
+	// if the range is nil, we assume it is full content change
+	if change.Range() == nil {
+		buf.Reset()
+		_, err := buf.WriteString(change.Text())
+		return err
+	}
+
+	lines := source.MakeSourceLines("", buf.Bytes())
+
+	startByte, err := ByteOffsetForPos(lines, change.Range().Start)
+	if err != nil {
+		return err
+	}
+	endByte, err := ByteOffsetForPos(lines, change.Range().End)
+	if err != nil {
+		return err
+	}
+
+	diff := endByte - startByte
+	if diff > 0 {
+		buf.Grow(diff)
+	}
+
+	beforeChange := make([]byte, startByte, startByte)
+	copy(beforeChange, buf.Bytes())
+	afterBytes := buf.Bytes()[endByte:]
+	afterChange := make([]byte, len(afterBytes), len(afterBytes))
+	copy(afterChange, afterBytes)
+
+	buf.Reset()
+
+	_, err = buf.Write(beforeChange)
+	if err != nil {
+		return err
+	}
+	_, err = buf.WriteString(change.Text())
+	if err != nil {
+		return err
+	}
+	_, err = buf.Write(afterChange)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}