feat: first version (V0) of the Workflow Management System (#3493)

# Description

Please include a summary of the changes and the related issue. Please
also include relevant motivation and context.

## Checklist before requesting a review

Please delete options that are not relevant.

- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my code
- [ ] I have commented hard-to-understand areas
- [ ] I have ideally added tests that prove my fix is effective or that
my feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged

## Screenshots (if appropriate):

Author: jacopo-chevallard

core/quivr_core/llm_tools/web_search_tools.py

@@ -45,8 +45,8 @@ def format_output(response: Any) -> List[Document]:
                 page_content=d["content"],
                 metadata={
                     **metadata,
-                    "file_name": d["url"],
-                    "original_file_name": d["url"],
+                    "file_name": d["url"] if "url" in d else "",
+                    "original_file_name": d["url"] if "url" in d else "",
                 },
             )
             for d in response

core/quivr_core/rag/entities/config.py

@@ -59,7 +59,7 @@ class DefaultRerankers(str, Enum):
     def default_model(self) -> str:
         # Mapping of suppliers to their default models
         return {
-            self.COHERE: "rerank-multilingual-v3.0",
+            self.COHERE: "rerank-v3.5",
             self.JINA: "jina-reranker-v2-base-multilingual",
             # self.MIXEDBREAD: "rmxbai-rerank-large-v1",
         }[self]
@@ -220,7 +220,7 @@ def set_api_key(self, force_reset: bool = False):
             return
 
         # Check if the corresponding API key environment variable is set
-        if not self.env_variable_name:
+        if force_reset or not self.env_variable_name:
             self.env_variable_name = (
                 f"{normalize_to_env_variable_name(self.supplier)}_API_KEY"
             )
@@ -230,7 +230,9 @@ def set_api_key(self, force_reset: bool = False):
 
         if not self.llm_api_key:
             logger.warning(f"The API key for supplier '{self.supplier}' is not set. ")
-            logger.warning(f"Please set the environment variable: '{self.env_variable_name}'. ")
+            logger.warning(
+                f"Please set the environment variable: '{self.env_variable_name}'. "
+            )
 
     def set_llm_model_config(self):
         # Automatically set context_length and tokenizer_hub based on the supplier and model

core/quivr_core/rag/prompts.py

@@ -61,23 +61,23 @@ def _define_custom_prompts() -> CustomPromptsDict:
         "- If no preferred language is provided, answer in the same language as the language used by the user.\n"
         "- You must use ONLY the provided context to answer the question. "
         "Do not use any prior knowledge or external information, even if you are certain of the answer.\n"
-        "- If you cannot provide an answer using ONLY the context provided, do not attempt to answer from your own knowledge."
-        "Instead, inform the user that the answer isn't available in the context and suggest using the available tools {tools}.\n"
+        # "- If you cannot provide an answer using ONLY the context provided, do not attempt to answer from your own knowledge."
+        # "Instead, inform the user that the answer isn't available in the context and suggest using the available tools {tools}.\n"
         "- Do not apologize when providing an answer.\n"
         "- Don't cite the source id in the answer objects, but you can use the source to answer the question.\n\n"
     )
 
     context_template = (
         "\n"
-        "- You have access to the following internal reasoning to provide an answer: {reasoning}\n"
+        # "- You have access to the following internal reasoning to provide an answer: {reasoning}\n"
         "- You have access to the following files to answer the user question (limited to first 20 files): {files}\n"
         "- You have access to the following context to answer the user question: {context}\n"
         "- Follow these user instruction when crafting the answer: {custom_instructions}\n"
         "- These user instructions shall take priority over any other previous instruction.\n"
-        "- Remember: if you cannot provide an answer using ONLY the provided context and CITING the sources, "
-        "inform the user that you don't have the answer and consider if any of the tools can help answer the question.\n"
-        "- Explain your reasoning about the potentiel tool usage in the answer.\n"
-        "- Only use binded tools to answer the question.\n"
+        # "- Remember: if you cannot provide an answer using ONLY the provided context and CITING the sources, "
+        # "inform the user that you don't have the answer and consider if any of the tools can help answer the question.\n"
+        # "- Explain your reasoning about the potentiel tool usage in the answer.\n"
+        # "- Only use binded tools to answer the question.\n"
         # "OFFER the user the possibility to ACTIVATE a relevant tool among "
         # "the tools which can be activated."
         # "Tools which can be activated: {tools}. If any of these tools can help in providing an answer "
@@ -88,6 +88,10 @@ def _define_custom_prompts() -> CustomPromptsDict:
     template_answer = (
         "Original task: {question}\n"
         "Rephrased and contextualized task: {rephrased_task}\n"
+        "Remember, you shall answer ALL questions and tasks.\n"
+        "Remember: if you cannot provide an answer using ONLY the provided context and CITING the sources, "
+        "just answer that you don't have the answer.\n"
+        "If the provided context contains contradictory or conflicting information, state so providing the conflicting information.\n"
     )
 
     RAG_ANSWER_PROMPT = ChatPromptTemplate.from_messages(
@@ -207,8 +211,8 @@ def _define_custom_prompts() -> CustomPromptsDict:
         "just reformulate them if needed and otherwise return them as is.\n"
         "- Remember, you shall NOT suggest or generate new tasks.\n"
         "- As an example, the user input 'What is Apple? Who is its CEO? When was it founded?' "
-        "shall be split into the questions 'What is Apple?', 'Who is the CEO of Apple?' and 'When was Apple founded?'.\n"
-        "- If no tasks are found, return the user input as is in the task.\n"
+        "shall be split into a list of tasks ['What is Apple?', 'Who is the CEO of Apple?', 'When was Apple founded?']\n"
+        "- If no tasks are found, return the user input as is in the task list.\n"
     )
 
     template_answer = "User input: {user_input}"