app assessment integration initial commit (#1044)

* app assessment integration initial commit

* added stored procs and changes to code report

---------

Co-authored-by: Aditya Bharadwaj <[email protected]>

Author: bharadwaj-aditya

assessment/assessment_engine.go

@@ -15,25 +15,34 @@
 package assessment
 
 import (
+	"context"
+	"fmt"
+
 	assessment "github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/collectors"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/profiles"
+	"go.uber.org/zap"
 )
 
 type assessmentCollectors struct {
-	sampleCollector     assessment.SampleCollector
-	infoSchemaCollector assessment.InfoSchemaCollector
+	sampleCollector        *assessment.SampleCollector
+	infoSchemaCollector    *assessment.InfoSchemaCollector
+	appAssessmentCollector *assessment.MigrationSummarizer
 }
 
-func PerformAssessment(conv *internal.Conv, sourceProfile profiles.SourceProfile) (utils.AssessmentOutput, error) {
+func PerformAssessment(conv *internal.Conv, sourceProfile profiles.SourceProfile, assessmentConfig map[string]string, projectId string) (utils.AssessmentOutput, error) {
 
 	logger.Log.Info("performing assessment")
+	logger.Log.Info(fmt.Sprintf("assessment config %+v", assessmentConfig))
+	logger.Log.Info(fmt.Sprintf("project id %+v", projectId))
+
+	ctx := context.Background()
 
 	output := utils.AssessmentOutput{}
 	// Initialize collectors
-	c, err := initializeCollectors(conv, sourceProfile)
+	c, err := initializeCollectors(conv, sourceProfile, assessmentConfig, projectId, ctx)
 	if err != nil {
 		logger.Log.Error("unable to initialize collectors")
 		return output, err
@@ -45,32 +54,86 @@ func PerformAssessment(conv *internal.Conv, sourceProfile profiles.SourceProfile
 	// Select the highest confidence output for each attribute
 	// Populate assessment struct
 
-	output.SchemaAssessment, err = performSchemaAssessment(c)
+	output.SchemaAssessment, err = performSchemaAssessment(ctx, c)
+
 	return output, err
 }
 
 // Initilize collectors. Take a decision here on which collectors are mandatory and which are optional
-func initializeCollectors(conv *internal.Conv, sourceProfile profiles.SourceProfile) (assessmentCollectors, error) {
+func initializeCollectors(conv *internal.Conv, sourceProfile profiles.SourceProfile, assessmentConfig map[string]string, projectId string, ctx context.Context) (assessmentCollectors, error) {
 	c := assessmentCollectors{}
 	sampleCollector, err := assessment.CreateSampleCollector()
 	if err != nil {
 		return c, err
 	}
-	c.sampleCollector = sampleCollector
+	c.sampleCollector = &sampleCollector
 	infoSchemaCollector, err := assessment.CreateInfoSchemaCollector(conv, sourceProfile)
 	if infoSchemaCollector.IsEmpty() {
 		return c, err
 	}
-	c.infoSchemaCollector = infoSchemaCollector
+	c.infoSchemaCollector = &infoSchemaCollector
+
+	//Initiialize App Assessment Collector
+
+	codeDirectory, exists := assessmentConfig["codeDirectory"]
+	if exists {
+		logger.Log.Info("initializing app collector")
+
+		mysqlSchema := ""   // TODO fetch from conv
+		spannerSchema := "" // TODO fetch from conv
+		mysqlSchemaPath, exists := assessmentConfig["mysqlSchemaPath"]
+		if exists {
+			logger.Log.Info(fmt.Sprintf("overriding mysql schema from file %s", mysqlSchemaPath))
+			mysqlSchema, err := utils.ReadFile(mysqlSchemaPath)
+			if err != nil {
+				logger.Log.Debug("error reading MySQL schema file:", zap.Error(err))
+			}
+			logger.Log.Debug(mysqlSchema)
+		}
+
+		spannerSchemaPath, exists := assessmentConfig["spannerSchemaPath"]
+		if exists {
+			logger.Log.Info(fmt.Sprintf("overriding spanner schema from file %s", spannerSchemaPath))
+			spannerSchema, err := utils.ReadFile(spannerSchemaPath)
+			if err != nil {
+				logger.Log.Debug("error reading Spanner schema file:", zap.Error(err))
+			}
+			logger.Log.Info(spannerSchema)
+		}
+
+		summarizer, err := assessment.NewMigrationSummarizer(ctx, nil, projectId, assessmentConfig["location"], mysqlSchema, spannerSchema, codeDirectory)
+		if err != nil {
+			logger.Log.Error("error initiating migration summarizer")
+			return c, err
+		}
+		c.appAssessmentCollector = summarizer
+		logger.Log.Info("initialized app collector")
+	} else {
+		logger.Log.Info("app code info unavailable")
+	}
+
 	return c, err
 }
 
-func performSchemaAssessment(collectors assessmentCollectors) (utils.SchemaAssessmentOutput, error) {
+func performSchemaAssessment(ctx context.Context, collectors assessmentCollectors) (utils.SchemaAssessmentOutput, error) {
 	schemaOut := utils.SchemaAssessmentOutput{}
 	schemaOut.SourceTableDefs, schemaOut.SpannerTableDefs = collectors.infoSchemaCollector.ListTables()
 	schemaOut.SourceIndexDef, schemaOut.SpannerIndexDef = collectors.infoSchemaCollector.ListIndexes()
 	schemaOut.Triggers = collectors.infoSchemaCollector.ListTriggers()
 	schemaOut.SourceColDefs, schemaOut.SpannerColDefs = collectors.infoSchemaCollector.ListColumnDefinitions()
 	schemaOut.StoredProcedureAssessmentOutput = collectors.infoSchemaCollector.ListStoredProcedures()
+
+	if collectors.appAssessmentCollector != nil {
+		logger.Log.Info("adding app assessment details")
+		codeAssessment, err := collectors.appAssessmentCollector.AnalyzeProject(ctx)
+
+		if err != nil {
+			logger.Log.Error("error analyzing project", zap.Error(err))
+			return schemaOut, err
+		}
+
+		logger.Log.Info(fmt.Sprintf("snippets %+v", codeAssessment.Snippets))
+		schemaOut.CodeSnippets = &codeAssessment.Snippets
+	}
 	return schemaOut, nil
 }

assessment/collectors/app_migration_analysis.go

@@ -17,7 +17,6 @@
 package assessment
 
 import (
-	"bufio"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -28,6 +27,7 @@ import (
 	assessment "github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/collectors/embeddings"
 	parser "github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/collectors/parser"
 	dependencyAnalyzer "github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/collectors/project_analyzer"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/utils"
 	. "github.com/GoogleCloudPlatform/spanner-migration-tool/assessment/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"go.uber.org/zap"
@@ -266,7 +266,7 @@ func ParseJSONWithRetries(model *genai.GenerativeModel, originalPrompt string, o
 func (m *MigrationSummarizer) fetchFileContent(filepath string) (string, error) {
 
 	// Read file content if not provided
-	content, err := readFile(filepath)
+	content, err := utils.ReadFile(filepath)
 	if err != nil {
 		logger.Log.Fatal("Failed read file: ", zap.Error(err))
 		return "", err
@@ -517,24 +517,6 @@ func getPromptForNonDAOClass(content, filepath string, methodChanges *string) st
 		%s`, filepath, content, *methodChanges)
 }
 
-func readFile(filepath string) (string, error) {
-	file, err := os.Open(filepath)
-	if err != nil {
-		return "", err
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	var content string
-	for scanner.Scan() {
-		content += scanner.Text() + "\n"
-	}
-	if err := scanner.Err(); err != nil {
-		return "", err
-	}
-	return content, nil
-}
-
 func getPromptForDAOClass(content, filepath string, methodChanges, oldSchema, newSchema *string) string {
 	//TODO: Move prompts to promt file.
 	return fmt.Sprintf(`

assessment/collectors/project_analyzer/file_dependency_analyzer.go

@@ -15,6 +15,7 @@
 package assessment
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
@@ -47,8 +48,7 @@ func (g *GoDependencyAnalyzer) getDependencyGraph(directory string) map[string]m
 		Dir:  (directory),
 	}
 
-	logger.Log.Debug(directory)
-
+	logger.Log.Debug(fmt.Sprintf("loading packages from directory: %s", directory))
 	pkgs, err := packages.Load(cfg, "./...")
 
 	if err != nil {