add model id & demo for watt-tool

Author: Judd

README.md

@@ -13,7 +13,7 @@ pure C++ implementation based on [@ggerganov](https://github.com/ggerganov)'s [g
 
 **What's New:**
 
-* 2025-02-19: MoE CPU offloading
+* 2025-02-19: MoE CPU offloading, tool calling with Watt-tool
 * 2025-02-17: [ggml updated](https://github.com/ggml-org/llama.cpp/tree/0f2bbe656473177538956d22b6842bcaa0449fab) again
 * 2025-02-10: [GPU acceleration](./docs/gpu.md) 🔥
 * 2025-01-25: MiniCPM Embedding & ReRanker

docs/models.md

@@ -35,6 +35,7 @@
         - v3: [Instruct-1B](https://huggingface.co/tiiuae/Falcon3-1B-Instruct), [Instruct-3B](https://huggingface.co/tiiuae/Falcon3-3B-Instruct), [Instruct-7B](https://huggingface.co/tiiuae/Falcon3-7B-Instruct), [Instruct-10B](https://huggingface.co/tiiuae/Falcon3-10B-Instruct)
     * [x] DeepSeek-R1-Distill-LlaMA: [8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B), [70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) (`-a DeepSeek-R1-Distill-LlaMA`)
     * [x] DeepHermes-3: [Llama-3-8B-Preview](https://huggingface.co/NousResearch/DeepHermes-3-Llama-3-8B-Preview) (Remember to user `-s ...`)
+    * [x] Watt-tool: [8B](https://huggingface.co/watt-ai/watt-tool-8B), [70B](https://huggingface.co/watt-ai/watt-tool-70B)
 
     For other models that using `LlamaForCausalLM` architecture, for example, [aiXcoder-7B](https://huggingface.co/aiXcoder/aixcoder-7b-base), try `-a Yi`.
 
@@ -94,7 +95,7 @@
 
     * [x] Codestral: [22B-v0.1](https://huggingface.co/mistralai/Codestral-22B-v0.1)
     * [x] Mistral-Nemo: [Nemo-Instruct-2407](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407)
-
+    * [x] Small: [Instruct-24B](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501)
 
 * Phi (`PhiForCausalLM`, `Phi3ForCausalLM`)
     * [x] [Phi-2](https://huggingface.co/microsoft/phi-2/tree/eb8bbd1d37d258ea74fb082c53346d33056a83d4)

docs/tool_calling.md

@@ -11,6 +11,7 @@ Demos of tool calling for these models are provided:
 * [NuminaMath](../scripts/tool_numinamath.py)
 * [LlaMA3-Groq-Tool-Use](../scripts/tool_groq.py)
 * [LlaMA 3.1](../scripts/tool_llama3.1.py)
+* [Watt-Tool](../scripts/tool_watt.py)
 
 ## Precondition
 

scripts/model_downloader.py

@@ -8,6 +8,8 @@ def get_model_url_on_modelscope(proj: str, fn: str, user: str = 'judd2024') -> s
 with open(os.path.join(binding.PATH_SCRIPTS, 'models.json'), encoding='utf-8') as f:
     all_models = json.load(f)
 
+DEF_STORAGE_DIR = '../quantized'
+
 def print_progress_bar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 60, fill = '█', printEnd = "\r", auto_nl = True):
     percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
     filledLength = int(length * iteration // total)
@@ -126,13 +128,13 @@ def enum_missing():
             for q in variants[s]['quantized'].keys():
                 quantized = variants[s]['quantized'][q]
                 all.add(quantized['url'].split('/')[1])
-    f = glob.glob(os.path.join('../quantized', '*.bin'))
+    f = glob.glob(os.path.join(DEF_STORAGE_DIR, '*.bin'))
     l = []
     for x in f:
         k = os.path.basename(x)
         if not k in all:
             l.append(k)
-    print(sorted(l))
+    print(f'not uploaded models: {sorted(l)}')
 
 def check_default():
     for m in all_models.keys():
@@ -141,4 +143,15 @@ def check_default():
             print(f"{m} default missing")
 
 if __name__ == '__main__':
+    import sys
+    args = sys.argv[1:]
+    if len(args) > 0:
+        if args[0] == 'check':
+            enum_missing()
+            check_default()
+            exit(0)
+
+        if args[0].startswith(':'):
+            print(preprocess_args(args, DEF_STORAGE_DIR))
+            exit(0)
     show()

scripts/models.json

@@ -2006,5 +2006,21 @@
                 }
             }
         }
+    },
+    "watt-tool": {
+        "brief": "watt-tool-8B is a fine-tuned language model based on LLaMa-3.1-8B-Instruct, optimized for tool usage and multi-turn dialogue.",
+        "default": "8b",
+        "license": "Apache License 2.0",
+        "variants": {
+            "8b": {
+                "default": "q8",
+                "quantized": {
+                    "q8": {
+                        "size": 8538752192,
+                        "url": "chatllm_quantized_watt-tool/watt-tool-8b.bin"
+                    }
+                }
+            }
+        }
     }
 }

scripts/tool_watt.py

@@ -0,0 +1,128 @@
+import json
+import re, sys
+from datetime import datetime
+
+from binding import PATH_BINDS
+
+import tool_definition
+from tool_definition import dispatch_tool
+
+# https://huggingface.co/watt-ai/watt-tool-8B
+
+FUNCTION_CALL_START = "["
+FUNCTION_CALL_CLOSE = "]"
+
+def convert_tool_def(func: dict) -> dict:
+    params = {}
+    required = set()
+    for p in func['parameters']:
+        params[p['name']] = {
+            "type": p['type'],
+            "description": p['description'],
+        }
+        if p['required']: required.add(p['name'])
+
+    desc = {
+        "name": func['name'],
+        "description": func['description'],
+        "arguments": {
+            "type": "dict",
+            "properties": params,
+            "required": list(required)
+        }
+    }
+    return desc
+
+SYS_PROMPT_TEMPLATE = """You are an expert in composing functions. You are given a question and a set of possible functions. Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+If none of the function can be used, point it out. If the given question lacks the parameters required by the function, also point it out.
+You should only return the function call in tools call sections.
+
+If you decide to invoke any of the function(s), you MUST put it in the format of [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)]
+You SHOULD NOT include any other text in the response.
+Here is a list of functions in JSON format that you can invoke.\n{functions}\n
+"""
+
+def build_system_prompt(functions: list[dict]):
+    s = SYS_PROMPT_TEMPLATE.format(functions=[convert_tool_def(f) for f in functions])
+    return s
+
+import chatllm
+from chatllm import ChatLLM
+
+def call_internal_tool(s: str) -> str:
+    print(f"[Use Builtin Tool]{s}")
+    return "Error: not implemented"
+
+def call_functions(s: str) -> str:
+    try:
+        r = []
+        for tool_name, code in parse_function_calls(s):
+            print(f'[Use Tool] {tool_name}')
+            observation = dispatch_tool(tool_name, code)
+            r.append(observation.text)
+        return '\n\n'.join(r)
+    except Exception as e:
+        print(f"error occurs: {e}")
+        return "failed to call the function"
+
+
+def parse_function_calls(s: str) -> list[tuple[str, dict]] | None:
+    try:
+        matches = re.findall(r'(\w+)\((.*?)\)', s)
+        if matches is None: return None
+
+        def parse_args(s: str) -> str:
+            r = []
+            for pair in s.split(', '):
+                k, v = pair.split('=')
+                r.append(f'"{k}": {v}')
+            return f"{{{','.join(r)}}}"
+
+        return [(match[0], json.loads(parse_args(match[1]))) for match in matches]
+    except:
+        return None
+
+class ToolChatLLM(ChatLLM):
+
+    chunk_acc = ''
+
+    def callback_print(self, s: str) -> None:
+
+        if self.chunk_acc == '':
+            if FUNCTION_CALL_START.startswith(s):
+                self.chunk_acc = s
+            else:
+                super().callback_print(s)
+
+            return
+
+        self.chunk_acc = (self.chunk_acc + s).strip()
+
+        if len(self.chunk_acc) < len(FUNCTION_CALL_START):
+            return
+
+        if not self.chunk_acc.startswith(FUNCTION_CALL_START):
+            super().callback_print(self.chunk_acc)
+            self.chunk_acc = ''
+            return
+
+    def callback_end(self) -> None:
+        s = self.chunk_acc
+        self.chunk_acc = ''
+        super().callback_end()
+
+        s = s.strip()
+        if len(s) < 1: return
+
+        if parse_function_calls(s) is not None:
+            rsp = call_functions(s)
+            self.tool_input(rsp)
+        else:
+            super().callback_print(s)
+
+    def call_tool(self, s: str) -> None:
+        rsp = call_internal_tool(s.strip())
+        self.tool_input(rsp)
+
+if __name__ == '__main__':
+    chatllm.demo_simple(sys.argv[1:] + ['-s', build_system_prompt(tool_definition._TOOL_DESCRIPTIONS)], ToolChatLLM, lib_path=PATH_BINDS)