🔍 feat: Mistral OCR API / Upload Files as Text (danny-avila#6274)

* refactor: move `loadAuthValues` to `~/services/Tools/credentials`

* feat: add createAxiosInstance function to configure axios with proxy support

* WIP: First pass mistral ocr

* refactor: replace getConvoFiles with getToolFiles for improved file retrieval logic

* refactor: improve document formatting in encodeAndFormat function

* refactor: remove unused resendFiles parameter from buildOptions function (this option comes from the agent config)

* fix: update getFiles call to include files with `text` property as well

* refactor: move file handling to `initializeAgentOptions`

* refactor: enhance addImageURLs method to handle OCR text and improve message formatting

* refactor: update message formatting to handle OCR text in various content types

* refactor: remove unused resendFiles property from compactAgentsSchema

* fix: add error handling for Mistral OCR document upload and logging

* refactor: integrate OCR capability into file upload options and configuration

* refactor: skip processing for text source files in delete request, as they are directly tied to database

* feat: add metadata field to ExtendedFile type and update PanelColumns and PanelTable components for localization and metadata handling

* fix: source icon styling

* wip: first pass, frontend file context agent resources

* refactor: add hover card with contextual information for File Context (OCR) in FileContext component

* feat: enhance file processing by integrating file retrieval for OCR resources in agent initialization

* feat: implement OCR config; fix: agent resource deletion for ocr files

* feat: enhance agent initialization by adding OCR capability check in resource priming

* ci: fix `~/config` module mock

* ci: add OCR property expectation in AppService tests

* refactor: simplify OCR config loading by removing environment variable extraction, to be done when OCR is actually performed

* ci: add unit test to ensure environment variable references are not parsed in OCR config

* refactor: disable base64 image inclusion in OCR request

* refactor: enhance OCR configuration handling by validating environment variables and providing defaults

* refactor: use file stream from disk for mistral ocr api

Author: danny-avila

api/app/clients/BaseClient.js

@@ -1121,9 +1121,13 @@ class BaseClient {
         return message;
       }
 
-      const files = await getFiles({
-        file_id: { $in: fileIds },
-      });
+      const files = await getFiles(
+        {
+          file_id: { $in: fileIds },
+        },
+        {},
+        {},
+      );
 
       await this.addImageURLs(message, files, this.visionMode);
 

api/app/clients/tools/util/handleTools.js

@@ -21,6 +21,7 @@ const {
 } = require('../');
 const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process');
 const { createFileSearchTool, primeFiles: primeSearchFiles } = require('./fileSearch');
+const { loadAuthValues } = require('~/server/services/Tools/credentials');
 const { createMCPTool } = require('~/server/services/MCP');
 const { loadSpecs } = require('./loadSpecs');
 const { logger } = require('~/config');
@@ -90,45 +91,6 @@ const validateTools = async (user, tools = []) => {
   }
 };
 
-const loadAuthValues = async ({ userId, authFields, throwError = true }) => {
-  let authValues = {};
-
-  /**
-   * Finds the first non-empty value for the given authentication field, supporting alternate fields.
-   * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
-   * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
-   */
-  const findAuthValue = async (fields) => {
-    for (const field of fields) {
-      let value = process.env[field];
-      if (value) {
-        return { authField: field, authValue: value };
-      }
-      try {
-        value = await getUserPluginAuthValue(userId, field, throwError);
-      } catch (err) {
-        if (field === fields[fields.length - 1] && !value) {
-          throw err;
-        }
-      }
-      if (value) {
-        return { authField: field, authValue: value };
-      }
-    }
-    return null;
-  };
-
-  for (let authField of authFields) {
-    const fields = authField.split('||');
-    const result = await findAuthValue(fields);
-    if (result) {
-      authValues[result.authField] = result.authValue;
-    }
-  }
-
-  return authValues;
-};
-
 /** @typedef {typeof import('@langchain/core/tools').Tool} ToolConstructor */
 /** @typedef {import('@langchain/core/tools').Tool} Tool */
 
@@ -348,7 +310,6 @@ const loadTools = async ({
 
 module.exports = {
   loadToolWithAuth,
-  loadAuthValues,
   validateTools,
   loadTools,
 };

api/app/clients/tools/util/index.js

@@ -1,9 +1,8 @@
-const { validateTools, loadTools, loadAuthValues } = require('./handleTools');
+const { validateTools, loadTools } = require('./handleTools');
 const handleOpenAIErrors = require('./handleOpenAIErrors');
 
 module.exports = {
   handleOpenAIErrors,
-  loadAuthValues,
   validateTools,
   loadTools,
 };

api/config/index.js

@@ -1,3 +1,4 @@
+const axios = require('axios');
 const { EventSource } = require('eventsource');
 const { Time, CacheKeys } = require('librechat-data-provider');
 const logger = require('./winston');
@@ -47,9 +48,24 @@ const sendEvent = (res, event) => {
   res.write(`event: message\ndata: ${JSON.stringify(event)}\n\n`);
 };
 
+function createAxiosInstance() {
+  const instance = axios.create();
+
+  if (process.env.proxy) {
+    const url = new URL(process.env.proxy);
+    instance.defaults.proxy = {
+      host: url.hostname,
+      protocol: url.protocol.replace(':', ''),
+    };
+  }
+
+  return instance;
+}
+
 module.exports = {
   logger,
   sendEvent,
   getMCPManager,
+  createAxiosInstance,
   getFlowStateManager,
 };

api/models/Conversation.js

@@ -15,19 +15,6 @@ const searchConversation = async (conversationId) => {
     throw new Error('Error searching conversation');
   }
 };
-/**
- * Searches for a conversation by conversationId and returns associated file ids.
- * @param {string} conversationId - The conversation's ID.
- * @returns {Promise<string[] | null>}
- */
-const getConvoFiles = async (conversationId) => {
-  try {
-    return (await Conversation.findOne({ conversationId }, 'files').lean())?.files ?? [];
-  } catch (error) {
-    logger.error('[getConvoFiles] Error getting conversation files', error);
-    throw new Error('Error getting conversation files');
-  }
-};
 
 /**
  * Retrieves a single conversation for a given user and conversation ID.
@@ -73,9 +60,46 @@ const deleteNullOrEmptyConversations = async () => {
   }
 };
 
+/**
+ * Retrieves files from a conversation that have either embedded=true
+ * or a metadata.fileIdentifier. Simplified and efficient query.
+ *
+ * @param {string} conversationId - The conversation ID
+ * @returns {Promise<MongoFile[]>} - Filtered array of matching file objects
+ */
+const getToolFiles = async (conversationId) => {
+  try {
+    const [result] = await Conversation.aggregate([
+      { $match: { conversationId } },
+      {
+        $project: {
+          files: {
+            $filter: {
+              input: '$files',
+              as: 'file',
+              cond: {
+                $or: [
+                  { $eq: ['$$file.embedded', true] },
+                  { $ifNull: ['$$file.metadata.fileIdentifier', false] },
+                ],
+              },
+            },
+          },
+          _id: 0,
+        },
+      },
+    ]).exec();
+
+    return result?.files || [];
+  } catch (error) {
+    logger.error('[getConvoEmbeddedFiles] Error fetching embedded files:', error);
+    throw new Error('Error fetching embedded files');
+  }
+};
+
 module.exports = {
   Conversation,
-  getConvoFiles,
+  getToolFiles,
   searchConversation,
   deleteNullOrEmptyConversations,
   /**

api/models/File.js

@@ -17,11 +17,13 @@ const findFileById = async (file_id, options = {}) => {
  * Retrieves files matching a given filter, sorted by the most recently updated.
  * @param {Object} filter - The filter criteria to apply.
  * @param {Object} [_sortOptions] - Optional sort parameters.
+ * @param {Object|String} [selectFields={ text: 0 }] - Fields to include/exclude in the query results.
+ *                                                   Default excludes the 'text' field.
  * @returns {Promise<Array<IMongoFile>>} A promise that resolves to an array of file documents.
  */
-const getFiles = async (filter, _sortOptions) => {
+const getFiles = async (filter, _sortOptions, selectFields = { text: 0 }) => {
   const sortOptions = { updatedAt: -1, ..._sortOptions };
-  return await File.find(filter).sort(sortOptions).lean();
+  return await File.find(filter).select(selectFields).sort(sortOptions).lean();
 };
 
 /**

api/server/controllers/agents/callbacks.js

@@ -10,8 +10,8 @@ const {
   ChatModelStreamHandler,
 } = require('@librechat/agents');
 const { processCodeOutput } = require('~/server/services/Files/Code/process');
+const { loadAuthValues } = require('~/server/services/Tools/credentials');
 const { saveBase64Image } = require('~/server/services/Files/process');
-const { loadAuthValues } = require('~/app/clients/tools/util');
 const { logger, sendEvent } = require('~/config');
 
 /** @typedef {import('@librechat/agents').Graph} Graph */

api/server/controllers/agents/client.js

@@ -223,14 +223,23 @@ class AgentClient extends BaseClient {
     };
   }
 
+  /**
+   *
+   * @param {TMessage} message
+   * @param {Array<MongoFile>} attachments
+   * @returns {Promise<Array<Partial<MongoFile>>>}
+   */
   async addImageURLs(message, attachments) {
-    const { files, image_urls } = await encodeAndFormat(
+    const { files, text, image_urls } = await encodeAndFormat(
       this.options.req,
       attachments,
       this.options.agent.provider,
       VisionModes.agents,
     );
     message.image_urls = image_urls.length ? image_urls : undefined;
+    if (text && text.length) {
+      message.ocr = text;
+    }
     return files;
   }
 
@@ -308,7 +317,21 @@ class AgentClient extends BaseClient {
         assistantName: this.options?.modelLabel,
       });
 
-      const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+      if (message.ocr && i !== orderedMessages.length - 1) {
+        if (typeof formattedMessage.content === 'string') {
+          formattedMessage.content = message.ocr + '\n' + formattedMessage.content;
+        } else {
+          const textPart = formattedMessage.content.find((part) => part.type === 'text');
+          textPart
+            ? (textPart.text = message.ocr + '\n' + textPart.text)
+            : formattedMessage.content.unshift({ type: 'text', text: message.ocr });
+        }
+      } else if (message.ocr && i === orderedMessages.length - 1) {
+        systemContent = [systemContent, message.ocr].join('\n');
+      }
+
+      const needsTokenCount =
+        (this.contextStrategy && !orderedMessages[i].tokenCount) || message.ocr;
 
       /* If tokens were never counted, or, is a Vision request and the message has files, count again */
       if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {

api/server/controllers/tools.js

@@ -10,7 +10,8 @@ const {
 const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
 const { processCodeOutput } = require('~/server/services/Files/Code/process');
 const { createToolCall, getToolCallsByConvo } = require('~/models/ToolCall');
-const { loadAuthValues, loadTools } = require('~/app/clients/tools/util');
+const { loadAuthValues } = require('~/server/services/Tools/credentials');
+const { loadTools } = require('~/app/clients/tools/util');
 const { checkAccess } = require('~/server/middleware');
 const { getMessage } = require('~/models/Message');
 const { logger } = require('~/config');

api/server/middleware/buildEndpointOption.js

@@ -10,7 +10,6 @@ const openAI = require('~/server/services/Endpoints/openAI');
 const agents = require('~/server/services/Endpoints/agents');
 const custom = require('~/server/services/Endpoints/custom');
 const google = require('~/server/services/Endpoints/google');
-const { getConvoFiles } = require('~/models/Conversation');
 const { handleError } = require('~/server/utils');
 
 const buildFunction = {
@@ -87,16 +86,8 @@ async function buildEndpointOption(req, res, next) {
 
     // TODO: use `getModelsConfig` only when necessary
     const modelsConfig = await getModelsConfig(req);
-    const { resendFiles = true } = req.body.endpointOption;
     req.body.endpointOption.modelsConfig = modelsConfig;
-    if (isAgents && resendFiles && req.body.conversationId) {
-      const fileIds = await getConvoFiles(req.body.conversationId);
-      const requestFiles = req.body.files ?? [];
-      if (requestFiles.length || fileIds.length) {
-        req.body.endpointOption.attachments = processFiles(requestFiles, fileIds);
-      }
-    } else if (req.body.files) {
-      // hold the promise
+    if (req.body.files && !isAgents) {
       req.body.endpointOption.attachments = processFiles(req.body.files);
     }
     next();

api/server/routes/files/files.js

@@ -16,7 +16,7 @@ const {
 } = require('~/server/services/Files/process');
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
-const { loadAuthValues } = require('~/app/clients/tools/util');
+const { loadAuthValues } = require('~/server/services/Tools/credentials');
 const { getAgent } = require('~/models/Agent');
 const { getFiles } = require('~/models/File');
 const { logger } = require('~/config');

api/server/services/AppService.js

@@ -1,4 +1,9 @@
-const { FileSources, EModelEndpoint, getConfigDefaults } = require('librechat-data-provider');
+const {
+  FileSources,
+  EModelEndpoint,
+  loadOCRConfig,
+  getConfigDefaults,
+} = require('librechat-data-provider');
 const { checkVariables, checkHealth, checkConfig, checkAzureVariables } = require('./start/checks');
 const { azureAssistantsDefaults, assistantsConfigSetup } = require('./start/assistants');
 const { initializeFirebase } = require('./Files/Firebase/initialize');
@@ -25,6 +30,7 @@ const AppService = async (app) => {
   const config = (await loadCustomConfig()) ?? {};
   const configDefaults = getConfigDefaults();
 
+  const ocr = loadOCRConfig(config.ocr);
   const filteredTools = config.filteredTools;
   const includedTools = config.includedTools;
   const fileStrategy = config.fileStrategy ?? configDefaults.fileStrategy;
@@ -57,6 +63,7 @@ const AppService = async (app) => {
   const interfaceConfig = await loadDefaultInterface(config, configDefaults);
 
   const defaultLocals = {
+    ocr,
     paths,
     fileStrategy,
     socialLogins,

api/server/services/AppService.spec.js

@@ -120,6 +120,7 @@ describe('AppService', () => {
         },
       },
       paths: expect.anything(),
+      ocr: expect.anything(),
       imageOutputType: expect.any(String),
       fileConfig: undefined,
       secureImageLinks: undefined,
@@ -588,4 +589,33 @@ describe('AppService updating app.locals and issuing warnings', () => {
       );
     });
   });
+
+  it('should not parse environment variable references in OCR config', async () => {
+    // Mock custom configuration with env variable references in OCR config
+    const mockConfig = {
+      ocr: {
+        apiKey: '${OCR_API_KEY_CUSTOM_VAR_NAME}',
+        baseURL: '${OCR_BASEURL_CUSTOM_VAR_NAME}',
+        strategy: 'mistral_ocr',
+        mistralModel: 'mistral-medium',
+      },
+    };
+
+    require('./Config/loadCustomConfig').mockImplementationOnce(() => Promise.resolve(mockConfig));
+
+    // Set actual environment variables with different values
+    process.env.OCR_API_KEY_CUSTOM_VAR_NAME = 'actual-api-key';
+    process.env.OCR_BASEURL_CUSTOM_VAR_NAME = 'https://actual-ocr-url.com';
+
+    // Initialize app
+    const app = { locals: {} };
+    await AppService(app);
+
+    // Verify that the raw string references were preserved and not interpolated
+    expect(app.locals.ocr).toBeDefined();
+    expect(app.locals.ocr.apiKey).toEqual('${OCR_API_KEY_CUSTOM_VAR_NAME}');
+    expect(app.locals.ocr.baseURL).toEqual('${OCR_BASEURL_CUSTOM_VAR_NAME}');
+    expect(app.locals.ocr.strategy).toEqual('mistral_ocr');
+    expect(app.locals.ocr.mistralModel).toEqual('mistral-medium');
+  });
 });