v1.0

local_inference/mlx.py

+from mlx_lm import load, generate
+
+model, tokenizer = load("mlx-community/SmolLM-1.7B-Instruct-8bit")
+response = generate(model, tokenizer, prompt="hello", verbose=True)
+
+print(response)
\ No newline at end of file

local_inference/transformers-js.js

+import { pipeline } from "@huggingface/transformers";
+
+// Create a text generation pipeline
+const generator = await pipeline(
+  "text-generation",
+  "HuggingFaceTB/SmolLM2-135M-Instruct",
+  { dtype: "q4f16", revision:'refs/pr/6' },
+);
+
+// Define the list of messages
+const messages = [
+  { role: "system", content: "You are a helpful assistant." },
+  { role: "user", content: "Rewrite the following: hello how r u?" },
+];
+
+// Generate a response
+const output = await generator(messages, { max_new_tokens: 128 });
+console.log(output[0].generated_text.at(-1).content);
+// "Hello, how's it going?"
\ No newline at end of file

model.properties

+# 模型编码
+modelCode=1120
+# 模型名称
+modelName=smollm_pytorch
+# 模型描述
+modelDescription=端侧小模型新星molLM2 1.7B击败Qwen 2.5 1.5B和Llama 3.2 1B，可以在不到2GB的VRAM上运行。
+# 应用场景
+appScenario=推理,训练,对话问答,制造,广媒,金融,能源,医疗,家居,教育
+# 框架类型
+frameType=pytorch

pre-training/README.md

+# Pre-training
+We use [nanotron](https://github.com/huggingface/nanotron/) library for training SmolLM and SmolLM2 base models.
+
+The scripts for training SmolLM v1 can be found in the `smollm1` folder. SmolLM2 has a similar architecture and setup but uses an improved data mixture that we curated and significantly longer training periods (11 trillion tokens for the 1.7B, 4 trillion for the 360M and 2 trillion for the 135M). We will upload the SmolLM2 configs soon.
+
+## Setup
+
+Please refer to [nanotron](https://github.com/huggingface/nanotron/) for detailed instructions on setting up your training environment and launching jobs.
+
+After setting up the environment and tokenizing the training datasets with [datatrove](https://github.com/huggingface/datatrove) (instructions available [here](https://github.com/huggingface/nanotron/blob/main/docs/nanoset.md#nanosets)), you can modify the configurations to match your number of nodes and local paths.
+
+Below is an example of launching SmolLM1 135M training on 1 node (you can change the DP value to 8 in the config and adjust the batch size) and run:
+
+```bash
+git clone https://github.com/huggingface/nanotron
+cd nanotron
+# follow installation
+CUDA_DEVICE_MAX_CONNECTIONS=1 torchrun --nproc_per_node=8 run_train.py --config-file smollm1/config_smollm1_135M.yaml
+```
+
+If you are working on a slurm cluster, you can modify the `launch.slurm` and launch the training with:
+
+```bash
+sbatch launch.slurm
+```
+> [!NOTE]
+> Don't forget to create the logs directory before launching the job:

pre-training/launch.slurm

+#!/bin/bash
+#SBATCH --job-name=smollm1-135M
+#SBATCH --nodes=4
+#SBATCH --gres=gpu:8
+#SBATCH --qos=high
+#SBATCH --output=./logs/train-%j.out
+#SBATCH --error=./logs/train-%j.err
+
+set -e
+
+TRAINER_PYTHON_FILE="run_train.py"
+CONFIG_PATH_YAML="smollm1/config_smollm1_135M.yaml"
+nvidia-smi
+
+# Show some environment variables
+echo python3 version = `python3 --version`
+echo "Python path: $(which python3)"
+echo "NCCL version: $(python -c "import torch;print(torch.cuda.nccl.version())")"
+echo "CUDA version: $(python -c "import torch;print(torch.version.cuda)")"
+
+echo "START TIME: $(date)"
+secs_to_human() {
+    echo "$(( ${1} / 3600 )):$(( (${1} / 60) % 60 )):$(( ${1} % 60 ))"
+}
+start=$(date +%s)
+echo "$(date -d @${start} "+%Y-%m-%d %H:%M:%S"): ${SLURM_JOB_NAME} start id=${SLURM_JOB_ID}\n"
+
+# SLURM stuff
+export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"`
+export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
+export MASTER_PORT=6000
+export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l`
+
+export CUDA_DEVICE_MAX_CONNECTIONS="1"
+
+echo "Number of nodes: $COUNT_NODE"
+echo "Hostnames: $HOSTNAMES"
+
+CMD=" $TRAINER_PYTHON_FILE \
+    --config-file $CONFIG_PATH_YAML \
+    "
+export LAUNCHER="torchrun \
+    --nproc_per_node 8 \
+    --nnodes $COUNT_NODE \
+    --node_rank $SLURM_PROCID \
+    --role $SLURMD_NODENAME: \
+    --max_restarts 0 \
+    --tee 3 \
+    "
+
+# Wait a random number between 0 and 1000 (milliseconds) to avoid too many concurrent requests to the hub
+random_milliseconds=$(( RANDOM % 1001 ))
+sleep_time=$(bc <<< "scale=3; $random_milliseconds / 1000")
+echo "Sleeping for $sleep_time seconds..."
+sleep $sleep_time
+
+srun $SRUN_ARGS -u bash -c "$LAUNCHER --node_rank $SLURM_PROCID --role $SLURMD_NODENAME: $CMD"
+
+echo "END TIME: $(date)"
\ No newline at end of file

pre-training/smollm1/config_smollm1_135M.yaml

+# SmolLM1 135M trained on 600B tokens
+checkpoints:
+  checkpoint_interval: 2000
+  checkpoints_path: checkpoints
+  checkpoints_path_is_shared_file_system: false
+  resume_checkpoint_path: null
+  save_final_state: false
+  save_initial_state: false
+data_stages:
+- data:
+    dataset:
+      dataset_folder: # paths to tokenized datasets
+        - datasets/fineweb-edu-dedup
+        - datasets/cosmopedia-v2
+        - datasets/python-edu
+        - datasets/open-web-math
+        - datasets/stackoverflow
+      dataset_weights:
+        - 0.7
+        - 0.15
+        - 0.08
+        - 0.06
+        - 0.01
+    num_loading_workers: 1
+    seed: 42
+  name: training stage
+  start_training_step: 1
+general:
+  benchmark_csv_path: null
+  consumed_train_samples: null
+  ignore_sanity_checks: true
+  project: smollm
+  run: smollm-135M
+  seed: 8
+  step: null
+logging:
+  iteration_step_info_interval: 1
+  log_level: info
+  log_level_replica: info
+model:
+  ddp_bucket_cap_mb: 25
+  dtype: bfloat16
+  init_method:
+    std: 0.0416 # 1/sqrt(hidden_size)
+  make_vocab_size_divisible_by: 1
+  model_config:
+    bos_token_id: 0
+    eos_token_id: 0
+    hidden_act: silu
+    hidden_size: 576
+    initializer_range: 0.02
+    intermediate_size: 1536
+    is_llama_config: true
+    max_position_embeddings: 2048
+    num_attention_heads: 9
+    num_hidden_layers: 30
+    num_key_value_heads: 3
+    pad_token_id: null
+    pretraining_tp: 1
+    rms_norm_eps: 1.0e-05
+    rope_scaling: null
+    rope_theta: 10000.0
+    tie_word_embeddings: true
+    use_cache: true
+    vocab_size: 49152
+optimizer:
+  accumulate_grad_in_fp32: true
+  clip_grad: 1.0
+  learning_rate_scheduler:
+    learning_rate: 0.003
+    lr_decay_starting_step: 250000
+    lr_decay_steps: 50000
+    lr_decay_style: 1-sqrt
+    lr_warmup_steps: 2500
+    lr_warmup_style: linear
+    min_decay_lr: 0
+  optimizer_factory:
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    adam_eps: 1.0e-08
+    name: adamW
+    torch_adam_is_fused: true
+  weight_decay: 0.01
+  zero_stage: 0
+parallelism:
+  dp: 32 # 4 nodes
+  expert_parallel_size: 1
+  pp: 1
+  pp_engine: 1f1b
+  recompute_layer: false
+  tp: 1
+  tp_linear_async_communication: true
+  tp_mode: REDUCE_SCATTER
+  tp_recompute_allgather: true
+profiler: null
+tokenizer:
+  tokenizer_max_length: null
+  tokenizer_name_or_path: HuggingFaceTB/cosmo2-tokenizer
+  tokenizer_revision: null
+tokens:
+  batch_accumulation_per_replica: 2
+  limit_test_batches: 0
+  limit_val_batches: 0
+  micro_batch_size: 8 # GBS = 8*2*32*sequence_length = 512*sequence_length = 1M tokens
+  sequence_length: 2048
+  train_steps: 600000
+  val_check_interval: -1
\ No newline at end of file

pre-training/smollm1/config_smollm1_1B.yaml

+# SmolLM1 1.7B trained on 1T tokens
+checkpoints:
+  checkpoint_interval: 2000
+  checkpoints_path: checkpoints
+  checkpoints_path_is_shared_file_system: false
+  resume_checkpoint_path: null
+  save_final_state: false
+  save_initial_state: false
+data_stages:
+- data:
+    dataset:
+      dataset_folder: # paths to tokenized datasets
+        - datasets/fineweb-edu-dedup
+        - datasets/cosmopedia-v2
+        - datasets/open-web-math
+        - datasets/starcoderdata-python
+        - datasets/stackoverflow
+      dataset_weights:
+        - 0.7
+        - 0.15
+        - 0.06
+        - 0.08
+        - 0.01
+    num_loading_workers: 1
+    seed: 42
+  name: training stage
+  start_training_step: 1
+- data:
+    dataset: # we change data mixture to use python-edu
+      dataset_folder: 
+        - datasets/fineweb-edu-dedup
+        - datasets/cosmopedia-v2
+        - datasets/open-web-math
+        - datasets/python-edu
+        - datasets/stackoverflow
+        - datasets/deepmind_mathematics
+      dataset_weights:
+        - 0.7
+        - 0.15
+        - 0.055
+        - 0.08
+        - 0.01
+        - 0.005
+    num_loading_workers: 1
+    seed: 42
+  name: training stage 2
+  start_training_step: 300000
+general:
+  benchmark_csv_path: null
+  consumed_train_samples: null
+  ignore_sanity_checks: true
+  project: smollm
+  run: smollm-1700M
+  seed: 8
+  step: null
+logging:
+  iteration_step_info_interval: 1
+  log_level: info
+  log_level_replica: info
+model:
+  ddp_bucket_cap_mb: 25
+  dtype: bfloat16
+  init_method:
+    std: 0.022097086912079608
+  make_vocab_size_divisible_by: 1
+  model_config:
+    bos_token_id: 0
+    eos_token_id: 0
+    hidden_act: silu
+    hidden_size: 2048
+    initializer_range: 0.02
+    intermediate_size: 8192
+    is_llama_config: true
+    max_position_embeddings: 2048
+    num_attention_heads: 32
+    num_hidden_layers: 24
+    num_key_value_heads: 32
+    pad_token_id: null
+    pretraining_tp: 1
+    rms_norm_eps: 1.0e-05
+    rope_scaling: null
+    rope_theta: 10000.0
+    tie_word_embeddings: true
+    use_cache: true
+    vocab_size: 49152
+optimizer:
+  accumulate_grad_in_fp32: true
+  clip_grad: 1.0
+  learning_rate_scheduler:
+    learning_rate: 0.0005
+    lr_decay_starting_step: 400000
+    lr_decay_steps: 100000
+    lr_decay_style: 1-sqrt
+    lr_warmup_steps: 2000
+    lr_warmup_style: linear
+    min_decay_lr: 0
+  optimizer_factory:
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    adam_eps: 1.0e-08
+    name: adamW
+    torch_adam_is_fused: true
+  weight_decay: 0.01
+  zero_stage: 0
+parallelism:
+  dp: 64 # 8 nodes
+  expert_parallel_size: 1
+  pp: 1
+  pp_engine: 1f1b
+  recompute_layer: false
+  tp: 1
+  tp_linear_async_communication: true
+  tp_mode: REDUCE_SCATTER
+  tp_recompute_allgather: true
+profiler: null
+tokenizer:
+  tokenizer_max_length: null
+  tokenizer_name_or_path: HuggingFaceTB/cosmo2-tokenizer
+  tokenizer_revision: null
+tokens:
+  batch_accumulation_per_replica: 4
+  limit_test_batches: 0
+  limit_val_batches: 0
+  micro_batch_size: 4 # GBS = 4*4*64*sequence_length = 1024*sequence_length = 2.1M tokens
+  sequence_length: 2048
+  train_steps: 500000
+  val_check_interval: -1
\ No newline at end of file

pre-training/smollm1/config_smollm1_360M.yaml

+# SmolLM1 360M trained on 600B tokens
+checkpoints:
+  checkpoint_interval: 2000
+  checkpoints_path: checkpoints
+  checkpoints_path_is_shared_file_system: false
+  resume_checkpoint_path: null
+  save_final_state: false
+  save_initial_state: false
+data_stages:
+- data:
+    dataset:
+      dataset_folder: # paths to tokenized datasets
+        - datasets/fineweb-edu-dedup
+        - datasets/cosmopedia-v2
+        - datasets/python-edu
+        - datasets/open-web-math
+        - datasets/stackoverflow
+      dataset_weights:
+        - 0.7
+        - 0.15
+        - 0.08
+        - 0.06
+        - 0.01
+    num_loading_workers: 1
+    seed: 42
+  name: training stage
+  start_training_step: 1
+general:
+  benchmark_csv_path: null
+  consumed_train_samples: null
+  ignore_sanity_checks: true
+  project: smollm
+  run: smollm-360M
+  seed: 8
+  step: null
+logging:
+  iteration_step_info_interval: 1
+  log_level: info
+  log_level_replica: info
+model:
+  ddp_bucket_cap_mb: 25
+  dtype: bfloat16
+  init_method:
+    std: 0.03227486121839514
+  make_vocab_size_divisible_by: 1
+  model_config:
+    bos_token_id: 0
+    eos_token_id: 0
+    hidden_act: silu
+    hidden_size: 960
+    initializer_range: 0.02
+    intermediate_size: 2560
+    is_llama_config: true
+    max_position_embeddings: 2048
+    num_attention_heads: 15
+    num_hidden_layers: 32
+    num_key_value_heads: 5
+    pad_token_id: null
+    pretraining_tp: 1
+    rms_norm_eps: 1.0e-05
+    rope_scaling: null
+    rope_theta: 10000.0
+    tie_word_embeddings: true
+    use_cache: true
+    vocab_size: 49152
+optimizer:
+  accumulate_grad_in_fp32: true
+  clip_grad: 1.0
+  learning_rate_scheduler:
+    learning_rate: 0.003
+    lr_decay_starting_step: 500000
+    lr_decay_steps: 100000
+    lr_decay_style: 1-sqrt
+    lr_warmup_steps: 5000
+    lr_warmup_style: linear
+    min_decay_lr: 0
+  optimizer_factory:
+    adam_beta1: 0.9
+    adam_beta2: 0.95
+    adam_eps: 1.0e-08
+    name: adamW
+    torch_adam_is_fused: true
+  weight_decay: 0.01
+  zero_stage: 0
+parallelism:
+  dp: 32
+  expert_parallel_size: 1
+  pp: 1
+  pp_engine: 1f1b
+  recompute_layer: false
+  tp: 1
+  tp_linear_async_communication: true
+  tp_mode: REDUCE_SCATTER
+  tp_recompute_allgather: true
+profiler: null
+tokenizer:
+  tokenizer_max_length: null
+  tokenizer_name_or_path: HuggingFaceTB/cosmo2-tokenizer
+  tokenizer_revision: null
+tokens:
+  batch_accumulation_per_replica: 2
+  limit_test_batches: 0
+  limit_val_batches: 0
+  micro_batch_size: 8
+  sequence_length: 2048
+  train_steps: 600000
+  val_check_interval: -1
\ No newline at end of file

smol_tools/README.md

+# smol-tools
+
+A collection of lightweight AI-powered tools built with LLaMA.cpp and small language models. These tools are designed to run locally on your machine without requiring expensive GPU resources. They can also run offline, without any internet connection.
+
+## Features
+
+### SmolSummarizer
+- Quick text summarization using SmolLM2-1.7B Instruct
+- Maintains key points while providing concise summaries
+- Able to reply to follow-up questions
+
+### SmolRewriter
+- Rewrites text to be more professional and approachable
+- Maintains the original message's intent and key points
+- Perfect for email and message drafting
+
+### SmolAgent
+- An AI agent that can perform various tasks through tool integration
+- Built-in tools include:
+  - Weather lookup
+  - Random number generation
+  - Current time
+  - Web browser control
+- Extensible tool system for adding new capabilities
+
+## Installation
+
+1. Clone the repository:
+
+```bash
+git clone https://github.com/huggingface/smollm.git
+cd smollm/smol_tools
+```
+
+2. Install dependencies:
+
+```bash
+uv venv --python 3.11
+source .venv/bin/activate
+uv pip install -r requirements.txt
+```
+
+If you're on mac, and you don't have tkinter installed, you can install it with:
+
+```bash
+brew install python-tk@3.11
+```
+
+For linux, you can install it with:
+
+```bash
+sudo apt-get install python3-tk
+```
+
+On Windows, when you install python you need to check the option to also install the tkinter library.
+
+## Usage
+
+### GUI Demo
+Run the Tkinter-based demo application:
+
+```bash
+python demo_tkinter.py
+```
+
+The demo provides a user-friendly interface with the following shortcuts:
+- `F1`: Open SmolDraft interface
+- `F2`: Summarize selected text
+- `F5`: Open SmolChat interface
+- `F10`: Open SmolAgent interface
+
+### Programmatic Usage
+
+```python
+from smol_tools.summarizer import SmolSummarizer
+from smol_tools.rewriter import SmolRewriter
+from smol_tools.agent import SmolToolAgent
+# Initialize tools
+summarizer = SmolSummarizer()
+rewriter = SmolRewriter()
+agent = SmolToolAgent()
+# Generate a summary
+for summary in summarizer.process("Your text here"):
+    print(summary)
+# Rewrite text
+for improved in rewriter.process("Your text here"):
+    print(improved)
+# Use the agent
+for response in agent.process("What's the weather in London?"):
+    print(response)
+```
+
+
+## Models
+
+The tools use the following models:
+- SmolSummarizer: SmolLM2-1.7B Instruct
+
+All models are quantized to 16-bit floating-point (F16) for efficient inference. Training was done on BF16, but in our tests, this format provides slower inference on Mac M-series chips.
+
+## License
+
+This project is licensed under the Apache License 2.0 - see the LICENSE file for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
\ No newline at end of file

smol_tools/requirements.txt

+tkmacosx>=1.0.5
+pynput>=1.7.7
+llama-cpp-python>=0.3.1
+pyperclip>=1.9.0
+transformers>=4.46.2
+pygments>=2.18.0
\ No newline at end of file

smol_tools/smol_tools/agent.py

+from .base import SmolTool
+from typing import Generator, List, Dict, Any, Callable
+import json
+import re
+from datetime import datetime
+import random
+from transformers import tool, CodeAgent
+import requests
+import webbrowser
+
+@tool
+def get_random_number_between(min: int, max: int) -> int:
+    """
+    Gets a random number between min and max.
+
+    Args:
+        min: The minimum number.
+        max: The maximum number.
+
+    Returns:
+        A random number between min and max.
+    """
+    return random.randint(min, max)
+
+
+@tool
+def get_weather(city: str) -> str:
+    """
+    Returns the weather forecast for a given city.
+
+    Args:
+        city: The name of the city.
+
+    Returns:
+        A string with a mock weather forecast.
+    """
+    url = 'https://wttr.in/{}?format=+%C,+%t'.format(city)
+    res = requests.get(url).text
+
+    return f"The weather in {city} is {res.split(',')[0]} with a high of {res.split(',')[1][:-2]} degrees Celsius."
+
+@tool
+def get_current_time() -> str:
+    """
+    This is a tool that returns the current time.
+    It returns the current time as HH:MM.
+    """
+    return f"The current time is {datetime.now().hour}:{datetime.now().minute}."
+
+@tool
+def open_webbrowser(url: str) -> str:
+    """
+    This is a tool that opens a web browser to the given website.
+    If the user asks to open a website or a browser, you should use this tool.
+
+    Args:
+        url: The url to open.
+    """
+    webbrowser.open(url)
+    return f"I opened {url.replace('https://', '').replace('www.', '')} in the browser."
+
+
+class SmolToolAgent(SmolTool):
+    def __init__(self):
+        self.tools = [get_random_number_between, get_current_time, open_webbrowser, get_weather]
+        self.toolbox = {tool.name: tool for tool in self.tools}
+        self.json_code_agent = CodeAgent(tools=self.tools, llm_engine=self.llm_engine, system_prompt=self._get_system_prompt())
+        super().__init__(
+            model_repo="andito/SmolLM2-1.7B-Instruct-F16-GGUF",
+            model_filename="smollm2-1.7b-8k-dpo-f16.gguf",
+            system_prompt=self._get_system_prompt(),
+            prefix_text=""
+        )
+
+    def llm_engine(self, messages, stop_sequences=["Task", "<|endoftext|>"]) -> str:
+        output = ""
+        for chunk in self.model.create_chat_completion(
+            messages=messages,
+            max_tokens=2048,
+            temperature=0.0,
+            top_p=1.0,
+            top_k=50,
+            repeat_penalty=1.0,
+            stream=True
+        ):
+            content = chunk['choices'][0]['delta'].get('content')
+            if content:
+                if content in ["<end_action>", "<|endoftext|>"]:
+                    break
+                output += content
+        return output
+
+    def _get_system_prompt(self) -> str:
+        return """You are an expert in composing functions. You are given a question and a set of possible functions. 
+Based on the question, you will need to make one or more function/tool calls to achieve the purpose. 
+If none of the functions can be used, point it out and refuse to answer. 
+If the given question lacks the parameters required by the function, also point it out.
+
+You have access to the following tools:
+<<tool_descriptions>>
+
+<<managed_agents_descriptions>>
+
+You can use imports in your code, but only from the following list of modules: <<authorized_imports>>
+
+The output MUST strictly adhere to the following format, and NO other text MUST be included.
+The example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make the tool calls an empty list '[]'.
+<tool_call>[
+{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},
+... (more tool calls as required)
+]</tool_call>"""
+
+    def _parse_response(self, text: str) -> List[Dict[str, Any]]:
+        pattern = r"<tool_call>(.*?)</tool_call>"
+        matches = re.findall(pattern, text, re.DOTALL)
+        if matches:
+            return json.loads(matches[0])
+        return text
+
+    def _call_tools(self, tool_calls: List[Dict[str, Any]]) -> List[str]:
+        tool_responses = []
+        for tool_call in tool_calls:
+            if tool_call["name"] in self.toolbox:
+                tool_responses.append(
+                    self.toolbox[tool_call["name"]](**tool_call["arguments"])
+                )
+            else:
+                tool_responses.append(f"Tool {tool_call['name']} not found.")
+        return tool_responses
+
+    def process(self, text: str) -> Generator[str, None, None]:
+        response = self.json_code_agent.run(text, return_generated_code=True)
+        # Parse and execute the tool calls
+        try:
+            tool_calls = self._parse_response(response)
+            if tool_calls in [response, [], ""]:
+                yield response
+                return
+            tool_responses = self._call_tools(tool_calls)
+        except Exception as e:
+            print("error", e)
+            yield response
+            return
+
+        # Yield each tool response
+        for response in tool_responses:
+            yield str(response) 
\ No newline at end of file

smol_tools/smol_tools/base.py

+from abc import ABC, abstractmethod
+from typing import Generator, List, Dict, Any, Union, Tuple
+from llama_cpp import Llama
+
+class SmolTool(ABC):
+    # Class-level cache for model instances
+    _model_cache: Dict[Tuple[str, str], Llama] = {}
+
+    def __init__(self, model_repo: str, model_filename: str, system_prompt: str, prefix_text: str = "", n_ctx: int = 8192):
+        self.system_prompt = system_prompt
+        self.prefix_text = prefix_text
+        
+        # Create a cache key from the model repo and filename
+        cache_key = (model_repo, model_filename)
+        
+        # Track if this is a new model load
+        is_new_model = cache_key not in self._model_cache
+        
+        # Try to get the model from cache, or create and cache a new one
+        if is_new_model:
+            print(f"Loading model {model_filename} from {model_repo}...")
+            self._model_cache[cache_key] = Llama.from_pretrained(
+                repo_id=model_repo,
+                filename=model_filename,
+                n_ctx=n_ctx,
+                verbose=False
+            )
+        
+        self.model = self._model_cache[cache_key]
+        
+        # Only warm up for newly loaded models
+        if is_new_model:
+            self._warm_up()
+
+    def _warm_up(self):
+        """Warm up the model with a test prompt"""
+        print(f"Warming up {self.__class__.__name__}...")
+        test_text = "This is a test message to warm up the model."
+        # Consume the generator to complete the warm-up
+        for _ in self.process(test_text):
+            pass
+        print(f"{self.__class__.__name__} ready!")
+
+    @abstractmethod
+    def process(self, text: str) -> Generator[str, None, None]:
+        """Process the input text and yield results as they're generated"""
+        pass
+
+    def _create_chat_completion(
+        self, 
+        messages: List[Dict[str, str]], 
+        temperature: float = 0.4,
+        top_p: float = 0.9,
+        top_k: int = 50,
+        repeat_penalty: float = 1.2,
+        max_tokens: int = 256
+    ) -> Generator[str, None, None]:
+        """Helper method to create chat completions with standard parameters"""
+        output = ""
+        for chunk in self.model.create_chat_completion(
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            repeat_penalty=repeat_penalty,
+            stream=True
+        ):
+            content = chunk['choices'][0]['delta'].get('content')
+            if content:
+                if content in ["<end_action>", "<|endoftext|>"]:
+                    break
+                output += content
+                yield output
\ No newline at end of file

smol_tools/smol_tools/chatter.py

+from .base import SmolTool
+from typing import Generator, List, Dict
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import os
+
+@dataclass
+class ChatMessage:
+    role: str  # "user" or "assistant"
+    content: str
+    timestamp: datetime
+
+    # Add methods to convert to/from dict for JSON serialization
+    def to_dict(self):
+        return {
+            'role': self.role,
+            'content': self.content,
+            'timestamp': self.timestamp.isoformat()
+        }
+
+    @classmethod
+    def from_dict(cls, data):
+        return cls(
+            role=data['role'],
+            content=data['content'],
+            timestamp=datetime.fromisoformat(data['timestamp'])
+        )
+
+class SmolChatter(SmolTool):
+    def __init__(self):
+        self.chat_history: List[ChatMessage] = []
+        self.chat_archive: Dict[str, List[ChatMessage]] = {}
+        self.current_chat_id = None
+        self.chats_dir = "saved_chats"
+        self._original_chat_state = None  # To track modifications
+        self.name = "SmolLM2-1.7B"
+        
+        # Create chats directory if it doesn't exist
+        if not os.path.exists(self.chats_dir):
+            os.makedirs(self.chats_dir)
+            
+        super().__init__(
+            model_repo="andito/SmolLM2-1.7B-Instruct-F16-GGUF",
+            model_filename="smollm2-1.7b-8k-dpo-f16.gguf",
+            system_prompt="You are a helpful AI assistant named SmolLM, trained by Hugging Face..",
+        )
+
+    def start_new_chat(self):
+        """Start a new chat with a unique ID"""
+        self.current_chat_id = datetime.now().strftime("%Y%m%d_%H%M%S")
+        self.chat_history = []
+        self._original_chat_state = None
+
+    def has_current_chat(self) -> bool:
+        """Check if there are any messages in the current chat"""
+        return len(self.chat_history) > 0
+
+    def save_current_chat(self, title: str = None, overwrite: bool = False):
+        """Save the current chat to disk if it has any messages"""
+        if not self.chat_history:
+            return
+            
+        if title:
+            # If overwriting, use existing chat_id if it matches the title
+            if not overwrite or self.current_chat_id != title:
+                self.current_chat_id = title
+        elif not self.current_chat_id:
+            self.current_chat_id = datetime.now().strftime("%Y%m%d_%H%M%S")
+            
+        # Convert chat history to serializable format
+        chat_data = {
+            'id': self.current_chat_id,
+            'messages': [msg.to_dict() for msg in self.chat_history]
+        }
+        
+        # Save to file
+        filename = f"{self.chats_dir}/chat_{self.current_chat_id}.json"
+        with open(filename, 'w') as f:
+            json.dump(chat_data, f)
+            
+        # Update original state to reflect saved state
+        self._original_chat_state = [msg.to_dict() for msg in self.chat_history]
+
+    def load_chat(self, chat_id: str):
+        """Load a specific chat from disk"""
+        filename = f"{self.chats_dir}/chat_{chat_id}.json"
+        try:
+            with open(filename, 'r') as f:
+                data = json.load(f)
+                self.current_chat_id = data['id']
+                self.chat_history = [ChatMessage.from_dict(msg) for msg in data['messages']]
+                # Store original state for modification tracking
+                self._original_chat_state = [msg.to_dict() for msg in self.chat_history]
+        except FileNotFoundError:
+            print(f"Chat {chat_id} not found")
+
+    def is_chat_modified(self) -> bool:
+        """Check if the current chat has been modified since loading"""
+        if self._original_chat_state is None:
+            # New chat that hasn't been saved yet
+            return len(self.chat_history) > 0
+            
+        current_state = [msg.to_dict() for msg in self.chat_history]
+        return current_state != self._original_chat_state
+
+    def get_saved_chats(self) -> List[str]:
+        """Get list of saved chat IDs"""
+        chats = []
+        for filename in os.listdir(self.chats_dir):
+            if filename.startswith('chat_') and filename.endswith('.json'):
+                chat_id = filename[5:-5]  # Remove 'chat_' prefix and '.json' suffix
+                chats.append(chat_id)
+        return sorted(chats, reverse=True)  # Most recent first
+
+    def _warm_up(self):
+        super()._warm_up()
+        self.clear_chat_history()
+
+    def process(self, text: str) -> Generator[str, None, None]:
+        # Add user message to history
+        self.chat_history.append(ChatMessage(
+            role="user",
+            content=text,
+            timestamp=datetime.now()
+        ))
+        
+        # Build messages including chat history
+        messages = [{"role": "system", "content": self.system_prompt}]
+        # Include last 5 messages for context
+        for msg in self.chat_history:
+            messages.append({"role": msg.role, "content": msg.content})
+
+        # Generate response
+        response = ""
+        for chunk in self._create_chat_completion(messages, max_tokens=1024):
+            response = chunk
+            yield chunk
+        
+        # Add assistant's response to history
+        self.chat_history.append(ChatMessage(
+            role="assistant",
+            content=response,
+            timestamp=datetime.now()
+        ))
+
+    def get_chat_history(self) -> List[ChatMessage]:
+        return self.chat_history
+    
+    def clear_chat_history(self):
+        self.chat_history = []
+
+    def get_current_chat_id(self) -> str:
+        """Get the ID of the current chat"""
+        return self.current_chat_id
\ No newline at end of file

smol_tools/smol_tools/rewriter.py

+from .base import SmolTool
+from typing import Generator
+
+class SmolRewriter(SmolTool):
+    def __init__(self):
+        super().__init__(
+            model_repo="andito/SmolLM2-1.7B-Instruct-F16-GGUF",
+            model_filename="smollm2-1.7b-8k-dpo-f16.gguf",
+            system_prompt="You are an AI writing assistant. Your task is to rewrite the user's email to make it more professional and approachable while maintaining its main points and key message. Do not return any text other than the rewritten message.",
+            prefix_text="Rewrite the message below to make it more professional and approachable while maintaining its main points and key message. Do not add any new information or return any text other than the rewritten message\nThe message:"
+        )
+
+    def process(self, text: str) -> Generator[str, None, None]:
+        messages = [
+            {"role": "system", "content": self.system_prompt},
+            {"role": "user", "content": f"{self.prefix_text}\n{text}"}
+        ]
+        yield from self._create_chat_completion(messages, temperature=0.4, repeat_penalty=1.0, top_k=0, max_tokens=1024)
\ No newline at end of file

smol_tools/smol_tools/summarizer.py

+from .base import SmolTool
+from typing import Generator, Optional
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List
+
+@dataclass
+class SummaryMessage:
+    role: str  # "user" or "assistant"
+    content: str
+    timestamp: datetime
+
+class SmolSummarizer(SmolTool):
+    def __init__(self):
+        self.name = "SmolLM2-1.7B"
+        
+        super().__init__(
+            model_repo="andito/SmolLM2-1.7B-Instruct-F16-GGUF",
+            model_filename="smollm2-1.7b-8k-dpo-f16.gguf",
+            system_prompt="Concisely summarize the main points of the input text in up to three sentences, focusing on key information and events.",
+        )
+
+    def process(self, text: str, question: Optional[str] = None) -> Generator[str, None, None]:
+        if question is None:
+            print("Summarizing text")
+            prompt = f"{self.prefix_text}\n{text}"
+            messages = [
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": prompt},
+                {"role": "assistant", "content": "This is a short summary of the text:"}
+            ]
+        else:
+            print("Answering question")
+            prompt = f"Original text:\n{text}\n\nQuestion: {question}"
+            messages = [
+                {"role": "user", "content": prompt},
+            ]
+
+        for chunk in self._create_chat_completion(messages, max_tokens=1024, temperature=0.1, top_p=0.9):
+            yield chunk

smol_tools/smol_tools/titler.py

+from .base import SmolTool
+from typing import Generator
+
+class SmolTitler(SmolTool):
+    def __init__(self):
+        super().__init__(
+            model_repo="andito/SmolLM2-1.7B-Instruct-F16-GGUF",
+            model_filename="smollm2-1.7b-8k-dpo-f16.gguf",
+            system_prompt="",
+            prefix_text="Create a title for this conversation:",
+        )
+
+    def process(self, text: str) -> Generator[str, None, None]:
+        messages = [
+            {"role": "user", "content": f"{self.prefix_text}\n{text}"}
+        ]
+        yield from self._create_chat_completion(messages, max_tokens=128, temperature=0.6, top_p=0.9, top_k=0, repeat_penalty=1.1)
\ No newline at end of file