fix: keep one url for the api and asset store

Author: ArslanSaleem

README.md

@@ -48,8 +48,7 @@ PandaETL is an open-source, no-code ETL (Extract, Transform, Load) tool designed
 3. Create a `.env` file in the frontend directory with the following:
 
    ```bash
-   NEXT_PUBLIC_API_URL=http://localhost:3000/api/v1
-   NEXT_PUBLIC_STORAGE_URL=http://localhost:3000/api/assets
+   NEXT_PUBLIC_API_URL=http://localhost:3000
    ```
 
    or copy the `.env.example` file to `.env`

backend/.env.example

@@ -3,6 +3,6 @@ PANDAETL_SERVER_URL="https://api.panda-etl.ai/" # optional
 API_SERVER_URL="https://api.domer.ai" # optional
 USE_OPENAI_EMBEDDINGS=false # optional
 OPENAI_API_KEY=sk-xxxxxxxxxxxx # optional
-CHROMA_BATCH_SIZE=5 # optional
+CHROMA_BATCH_SIZE=500 # optional
 MAX_FILE_SIZE=20971520 # optional
-PANDAETL_API_KEY=xxx-xxx-xxx-xxx # optional if you already have a PandaETL api key
+PANDAETL_API_KEY=xxx-xxx-xxx-xxx # optional if you already have a PandaETL api key

backend/app/processing/file_preprocessing.py

@@ -44,7 +44,7 @@ def process_segmentation(project_id: int, asset_id: int, asset_file_name: str):
         vectorstore.add_docs(
             docs=docs,
             metadatas=metadatas,
-            batch_size=100
+            batch_size=settings.chroma_batch_size
         )
 
         project_repository.update_asset_content_status(

backend/app/processing/process_queue.py

@@ -406,4 +406,4 @@ def vectorize_extraction_process_step(project_id: int, process_step_id: int, fil
     ]
 
     # Add documents to vectorstore
-    vectorstore.add_docs(docs=docs, metadatas=metadatas, batch_size=100)
+    vectorstore.add_docs(docs=docs, metadatas=metadatas, batch_size=settings.chroma_batch_size)

backend/tests/processing/test_process_queue.py

@@ -216,7 +216,7 @@ def test_vectorize_extraction_process_step_single_reference(mock_chroma_db):
     mock_vectorstore.add_docs.assert_called_once_with(
         docs=expected_docs,
         metadatas=expected_metadatas,
-        batch_size=100
+        batch_size=5
     )
 
 @patch('app.processing.process_queue.ChromaDB')
@@ -263,7 +263,7 @@ def test_vectorize_extraction_process_step_multiple_references_concatenation(moc
     mock_vectorstore.add_docs.assert_called_once_with(
         docs=expected_docs,
         metadatas=expected_metadatas,
-        batch_size=100
+        batch_size=5
     )
 
 @patch('app.processing.process_queue.ChromaDB')  # Replace with the correct module path

frontend/.env.example

@@ -1,5 +1,4 @@
-NEXT_PUBLIC_API_URL=http://localhost:3000/api/v1
-NEXT_PUBLIC_STORAGE_URL=http://localhost:3000/api/assets
+NEXT_PUBLIC_API_URL=http://localhost:3000
 NEXT_PUBLIC_MIXPANEL_TOKEN=f2e8a71ab2bde33ebf346c5abf6ba9fa
 NEXT_PUBLIC_ROLLBAR_ACCESS_TOKEN=0df0bee895044430880278e2b2a5b2d2
 # NEXT_PUBLIC_BACKEND_URL=http://backend:5328 # Uncomment this if you're working with a docker setup

frontend/next.config.mjs

@@ -1,14 +1,23 @@
 const nextConfig = {
   swcMinify: false, // TODO - track and remove this later: https://github.com/wojtekmaj/react-pdf/issues/1822
   async rewrites() {
+    const backendUrl =
+      process.env.NEXT_PUBLIC_BACKEND_URL || process.env.NEXT_PUBLIC_API_URL;
+
+    if (!backendUrl) {
+      console.warn(
+        "Warning: No backend URL provided. API routes may not work correctly."
+      );
+    }
+
     return [
-      // {
-      //   source: "/api/:path*",
-      //   destination: "http://localhost:5328/:path*",
-      // },
+      {
+        source: "/api/:path*",
+        destination: `${backendUrl}/v1/:path*`,
+      },
       {
         source: "/assets/:path*",
-        destination: "http://localhost:5328/assets/:path*",
+        destination: `${backendUrl}/assets/:path*`,
       },
     ];
   },

frontend/src/constants.ts

@@ -1,3 +1,3 @@
-export const BASE_API_URL = process.env.NEXT_PUBLIC_API_URL;
-export const BASE_STORAGE_URL = process.env.NEXT_PUBLIC_STORAGE_URL;
+export const BASE_API_URL = `${process.env.NEXT_PUBLIC_API_URL}/v1`;
+export const BASE_STORAGE_URL = `${process.env.NEXT_PUBLIC_API_URL}/assets`;
 export const MAX_FILE_SIZE = 20 * 1024 * 1024; // 20MB in bytes

frontend/src/middleware.ts

@@ -11,6 +11,7 @@ export async function middleware(request: NextRequest) {
     const dockerBackendUrl = process.env.NEXT_PUBLIC_BACKEND_URL;
 
     if (dockerBackendUrl) {
+      console.log(dockerBackendUrl);
       const response = await axios.get<{ data: APIKeyData }>(
         `${dockerBackendUrl}/v1/user/get-api-key`
       );