diff --git a/.gitmodules b/.gitmodules index 234ac82..ed8d662 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "gtk-server/uthash"] path = gtk-server/uthash url = https://github.com/troydhanson/uthash.git +[submodule "llama/llama.cpp"] + path = llama/llama.cpp + url = https://github.com/ggerganov/llama.cpp diff --git a/gtk-server/uthash b/gtk-server/uthash index 41c357f..6d85739 160000 --- a/gtk-server/uthash +++ b/gtk-server/uthash @@ -1 +1 @@ -Subproject commit 41c357fd74ade4f4b4822c4407d2f51c4558e18d +Subproject commit 6d8573997c21f24c7e4ec9e48734b44f384170a1 diff --git a/include/module.h b/include/module.h index c7753b2..5a81801 100644 --- a/include/module.h +++ b/include/module.h @@ -120,13 +120,23 @@ int sblib_func_exec(int index, int param_count, slib_par_t *params, var_t *retva /** * @ingroup modlib * - * executes a function + * free resources associated with the variable * * @param cls_id the variable class identifier * @param id the variable instance identifier */ int sblib_free(int cls_id, int id); +/** + * @ingroup modlib + * + * registers a fresh id to replace the given id + * + * @param cls_id the variable class identifier + * @param id the variable instance identifier + */ +int sblib_refresh_id(int cls_id, int id); + /** * @ingroup modlib * diff --git a/include/param.cpp b/include/param.cpp index 135e5af..5a5fea1 100644 --- a/include/param.cpp +++ b/include/param.cpp @@ -589,14 +589,14 @@ void v_create_func(var_p_t map, const char *name, method cb) { var_p_t v_func = map_add_var(map, name, 0); v_func->type = V_FUNC; v_func->v.fn.cb = cb; - v_func->v.fn.mcb = NULL; + v_func->v.fn.mcb = nullptr; v_func->v.fn.id = 0; } void v_create_callback(var_p_t map, const char *name, callback cb) { var_p_t v_func = map_add_var(map, name, 0); v_func->type = V_FUNC; - v_func->v.fn.cb = NULL; + v_func->v.fn.cb = nullptr; v_func->v.fn.mcb = cb; v_func->v.fn.id = 0; } diff --git a/include/var.h b/include/var.h index 16e0f4d..02a5a0a 100644 --- a/include/var.h +++ b/include/var.h @@ -80,7 +80,7 @@ typedef struct var_s { // associative array/map struct { - // pointer the map structure + // pointer to the map structure void *map; uint32_t count; @@ -132,7 +132,7 @@ typedef struct var_s { // non-zero if constant uint8_t const_flag; - // whether help in pooled memory + // whether held in pooled memory uint8_t pooled; } var_t; @@ -154,7 +154,7 @@ var_t *v_new(void); * * @return a newly created var_t array of the given size */ -void v_new_array(var_t *var, unsigned size); +void v_new_array(var_t *var, uint32_t size); /** * @ingroup var diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt new file mode 100644 index 0000000..a61dd32 --- /dev/null +++ b/llama/CMakeLists.txt @@ -0,0 +1,176 @@ +cmake_minimum_required(VERSION 3.15) +project(llm C CXX) + +# clang-check ../*.cpp +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_C_STANDARD 11) + +# ----------------------------- +# Path to llama.cpp +# ----------------------------- +set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp) + +# ----------------------------- +# FORCE static builds +# ----------------------------- +# Disable all shared libraries globally +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) + +# llama.cpp specific static settings +set(LLAMA_STATIC ON CACHE BOOL "" FORCE) +set(LLAMA_SHARED OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_LLAMA_SHARED OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_GGML_SHARED OFF CACHE BOOL "" FORCE) +set(LLAMA_SERVER_BUILD OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + +# ggml specific static settings +set(GGML_STATIC ON CACHE BOOL "" FORCE) +set(GGML_SHARED OFF CACHE BOOL "" FORCE) +set(GGML_BUILD_SHARED OFF CACHE BOOL "" FORCE) +set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# ------------------------------- +# Define backend options +# ------------------------------- +set(LLAMA_BACKEND "AUTO" CACHE STRING "Select llama.cpp backend: AUTO, CPU, GPU, CUDA") +set_property(CACHE LLAMA_BACKEND PROPERTY STRINGS AUTO CPU GPU CUDA) + +# +# sudo apt install nvidia-open cuda-toolkit +# + +# ------------------------------- +# Disable all accelerators by default +# ------------------------------- +set(GGML_OPENMP OFF CACHE BOOL "" FORCE) +set(GGML_CUDA OFF CACHE BOOL "" FORCE) +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(GGML_METAL OFF CACHE BOOL "" FORCE) # Apple GPU API +set(GGML_OPENCL OFF CACHE BOOL "" FORCE) # Cross platform GPU API (AMD/Intel) +set(GGML_KOMPUTE OFF CACHE BOOL "" FORCE) # Vulcan/more modern than OpenCL +set(GGML_SYCL OFF CACHE BOOL "" FORCE) # Intel +set(GGML_ACCELERATE OFF CACHE BOOL "" FORCE) # Apple + +# ------------------------------- +# Configure backends based on LLAMA_BACKEND +# ------------------------------- +include(CheckLanguage) + +if(LLAMA_BACKEND STREQUAL "CPU") + message(STATUS "llama.cpp backend: CPU-only") + set(GGML_NATIVE ON CACHE BOOL "" FORCE) # enable CPU SIMD optimizations +elseif(LLAMA_BACKEND STREQUAL "GPU") + message(STATUS "llama.cpp backend: GPU (non-CUDA)") + set(GGML_OPENMP ON CACHE BOOL "" FORCE) # parallel CPU fallback + # GPU non-CUDA options can be added here in the future +elseif(LLAMA_BACKEND STREQUAL "CUDA") + message(STATUS "llama.cpp backend: CUDA") + check_language(CUDA) + if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + set(GGML_CUDA ON CACHE BOOL "" FORCE) + else() + message(FATAL_ERROR "CUDA backend requested but nvcc not found") + endif() +elseif(LLAMA_BACKEND STREQUAL "AUTO") + message(STATUS "llama.cpp backend: AUTO") + check_language(CUDA) + if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + set(GGML_CUDA ON CACHE BOOL "" FORCE) + message(STATUS "CUDA detected – enabling GGML_CUDA") + else() + set(GGML_OPENMP ON CACHE BOOL "" FORCE) + set(GGML_NATIVE ON CACHE BOOL "" FORCE) + message(STATUS "CUDA not found – using CPU/OpenMP") + endif() +else() + message(FATAL_ERROR "Invalid LLAMA_BACKEND value: ${LLAMA_BACKEND}") +endif() + +# ----------------------------- +# Add llama.cpp subdirectories +# ----------------------------- +add_subdirectory(${LLAMA_DIR}/ggml) +add_subdirectory(${LLAMA_DIR}) + +# ----------------------------- +# Build plugin as a shared library (.so) +# ----------------------------- +set(PLUGIN_SOURCES + main.cpp + llama-sb.cpp + ../include/param.cpp + ../include/hashmap.cpp + ../include/apiexec.cpp +) + +add_library(llm SHARED ${PLUGIN_SOURCES}) + +target_include_directories(llm PRIVATE + ${LLAMA_DIR}/include + ${LLAMA_DIR}/ggml/include + ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +target_link_libraries(llm PRIVATE + llama + ggml + # force dynamic libm + -Wl,-Bdynamic,-lm +) + +# Include all static code into plugin +target_link_options(llm PRIVATE + -Wl,--whole-archive + $ + $ + -Wl,--no-whole-archive +) + +# Ensure position-independent code for .so +set_target_properties(llm PROPERTIES + POSITION_INDEPENDENT_CODE ON + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib +) + +# ------------------------------------------------------------------ +# Android native library +# ------------------------------------------------------------------ +if (ANDROID) + set(GGML_LLAMAFILE OFF CACHE BOOL "" FORCE) + set(GGML_BLAS OFF CACHE BOOL "" FORCE) + + # CMake sets ANDROID when using the Android toolchain + # Re‑use the same source files for the Android .so + add_library(llm_android SHARED + main.cpp + llama-sb.cpp + ../include/param.cpp + ../include/hashmap.cpp + ../include/apiexec.cpp + ) + + # Optional: set the SONAME / versioning if you need it + set_target_properties(llm_android PROPERTIES + OUTPUT_NAME "libllm" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${ANDROID_ABI}") + + target_link_libraries(llm_test PRIVATE + log + llm + llama + ggml + ) + + # Export the location so Gradle can copy it later + set(MY_NATIVE_LIB_PATH "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${ANDROID_ABI}/libllm.so") +endif() diff --git a/llama/RAG.md b/llama/RAG.md new file mode 100644 index 0000000..6c69c06 --- /dev/null +++ b/llama/RAG.md @@ -0,0 +1,323 @@ +# notcurses RAG — C++ Library Expert via llama.cpp + +A self-contained RAG (Retrieval-Augmented Generation) pipeline in C++17 +that turns a GGUF inference model into a focused expert on any C/C++ library. +Demonstrated here with [notcurses](https://github.com/dankamongmen/notcurses) +but works with any header-based library. + +No fixed limits on chunk count, chunk length, or embedding dimension. +No Python, no vector database daemon, no external dependencies beyond llama.cpp. + +--- + +## How it works + +``` +INDEXING (one-time offline) +──────────────────────────────────────────────────────────────── +notcurses headers + │ + ▼ +chunk_headers ← semantic chunker, outputs chunks.jsonl + │ + ▼ +rag_index ← embeds each chunk via qwen3-embedding-0.6b-q4_k_m.gguf + │ + ▼ +notcurses.db ← binary vector store (embeddings + text) + + +RUNTIME (each query) +──────────────────────────────────────────────────────────────── +user query + │ + ▼ +rag_retrieve() ← embeds query, cosine similarity against db + │ ← skips chunks already seen this session + ▼ +new top-k chunks ← most relevant unseen API fragments + │ + ▼ +prompt assembly ← system + prior history + new context + query + │ + ▼ +Qwen3 inference ← <|think|> reasoning + final answer + │ + ▼ +history ← appended for next turn (KV cache intact) +``` + +--- + +## Files + +| File | Purpose | +|---|---| +| `chunk_headers.cpp` | Parses C/C++ headers into semantic chunks, outputs `.jsonl` | +| `rag_index.cpp` | Reads `.jsonl`, embeds each chunk, saves binary `.db` | +| `rag.hpp` | Single-header C++17 runtime — load db, session, retrieve | +| `example.cpp` | Full pipeline wired together, multi-turn query loop | + +--- + +## Dependencies + +- [llama.cpp](https://github.com/ggerganov/llama.cpp) — `libllama` + `llama.h` +- A GGUF **inference model** — tested with `Qwen3.5-9B-Q4_K_M.gguf` +- A GGUF **embedding model** — `qwen3-embedding-0.6b-q4_k_m.gguf` +- C++17 compiler (gcc 8+, clang 7+, MSVC 2019+) + +--- + +## Build + +```bash +c++ -std=c++17 -o chunk_headers chunk_headers.cpp +c++ -std=c++17 -o rag_index rag_index.cpp -lllama -lm +c++ -std=c++17 -o example example.cpp -lllama -lm +``` + +If llama.cpp is not on your system library path: + +```bash +c++ -std=c++17 -o rag_index rag_index.cpp \ + -I/path/to/llama.cpp/include \ + -L/path/to/llama.cpp/build -lllama -lm +``` + +--- + +## Usage + +### Step 1 — Chunk the headers (one-time) + +```bash +./chunk_headers notcurses/include/notcurses/ > chunks.jsonl +``` + +Accepts a single file or a directory (walked recursively). +Multiple paths can be given: + +```bash +./chunk_headers include/foo.h include/bar.h src/examples/ > chunks.jsonl +``` + +Handles `.h`, `.hpp`, `.c`, `.cpp`. Inspect before indexing: + +```bash +head -5 chunks.jsonl | python3 -m json.tool +``` + +### Step 2 — Embed and index (one-time) + +```bash +./rag_index \ + --model qwen3-embedding-0.6b-q4_k_m.gguf \ + --input chunks.jsonl \ + --output notcurses.db +``` + +Takes a few minutes for a large corpus. The `.db` is reusable +until the library changes. + +### Step 3 — Run + +```bash +./example \ + --model Qwen3.5-9B-Q4_K_M.gguf \ + --embed qwen3-embedding-0.6b-q4_k_m.gguf \ + --db notcurses.db +``` + +``` +notcurses expert ready. ctrl+d to quit. + +you: how do I create a plane and render text into it? +assistant: ... + +you: what options does it take? ← follow-up; no repeated context +assistant: ... +``` + +--- + +## Using rag.hpp in your own project + +Single-header, stb-style. In **one** `.cpp` file: + +```cpp +#define RAG_IMPLEMENTATION +#include "rag.hpp" +``` + +All other files that need the types: + +```cpp +#include "rag.hpp" +``` + +### Minimal integration + +```cpp +// startup +RagDB db; +rag_load(db, "notcurses.db"); + +RagSession session; +session.init(db.size(), 8192); // n_chunks, your n_ctx +session.score_threshold = 0.60f; + +// each turn +std::string context = rag_retrieve(db, embed_ctx, embed_model, + user_query, 5, session); +// context is empty string if nothing new/relevant was found +// build prompt with context and hand to your inference context +``` + +### Stateless retrieval (no deduplication) + +```cpp +std::string context = rag_retrieve(db, embed_ctx, embed_model, + user_query, 5); +``` + +### API + +```cpp +// Load .db file (version 2). Returns true on success. +bool rag_load(RagDB &db, const std::string &path); + +// Retrieve with session deduplication + token budget. +// Returns context string ready to inject into prompt. +// Empty string if nothing new or relevant was found. +std::string rag_retrieve(const RagDB &db, + llama_context *embed_ctx, + llama_model *embed_model, + const std::string &query, + int top_k, + RagSession &session); + +// Stateless overload — no deduplication. +std::string rag_retrieve(const RagDB &db, + llama_context *embed_ctx, + llama_model *embed_model, + const std::string &query, + int top_k); +``` + +### RagSession fields + +```cpp +struct RagSession { + std::vector seen; // one bit per chunk, sized to db + int tokens_used = 0; // running token estimate + int tokens_max = 0; // your n_ctx ceiling + float score_threshold = 0.60f; // skip weak matches + + void init(int n_chunks, int ctx_size); + void reset(); // start a fresh conversation +}; +``` + +--- + +## Chunking strategy + +`chunk_headers` uses a state machine that keeps each **semantic unit** +together as one chunk: + +- Block comment (`/* ... */`) + following declaration +- `//` line comments + following declaration +- `typedef struct` / `typedef enum` entire body +- Consecutive `#define` macro groups +- Multi-line function signatures + +Example — this stays as one chunk: + +```c +// ncplane_create() - create a new plane as a child of 'n'. +// 'nopts' may be NULL for defaults. Returns NULL on error. +struct ncplane* ncplane_create(struct ncplane *n, + const struct ncplane_options *nopts); +``` + +--- + +## Session deduplication + +The KV cache is not cleared between turns, so the model already has +earlier chunks in memory. `RagSession` tracks which chunks have been +injected and skips them on subsequent turns: + +``` +Turn 1: retrieved chunks [42, 17, 83] → all new → inject all +Turn 2: retrieved chunks [42, 55, 17] → 42,17 seen → inject only [55] +Turn 3: retrieved chunks [7, 14, 55] → 55 seen → inject [7, 14] +``` + +Context window grows efficiently — no repeated API reference, and the +model remembers everything already seen via the intact KV cache. + +--- + +## Adapting to other libraries + +Change only the input to `chunk_headers`: + +| Library | Input | +|---|---| +| stb (stb_image, stb_truetype ...) | single `.h` file | +| SDL2 / OpenGL / Vulkan | `include/` directory | +| Your own engine | any `.h` / `.hpp` mix | +| Spring / Java | extend chunker for Javadoc + `.java` | + +Re-run steps 1 and 2 to produce a new `.db`. Runtime code unchanged. +Multiple `.db` files can be loaded and queried independently. + +--- + +## .db file format (version 2) + +Variable-length fields — no wasted padding. + +``` +Header (16 bytes): + uint32 magic = 0x52414744 ("RAGD") + uint32 version = 2 + uint32 n_chunks + uint32 embed_dim + +Per chunk: + uint32 text_len + char[] text (text_len bytes, no null) + uint16 source_len + char[] source (source_len bytes, no null) + uint8 type_len + char[] type (type_len bytes, no null) + float[] embedding (embed_dim × 4 bytes) +``` + +--- + +## GPU memory + +On an 8 GB GPU with `Qwen3.5-9B-Q4_K_M`: + +| Component | VRAM | +|---|---| +| Inference model (Q4_K_M 9B) | ~5.5 GB | +| Embedding model (nomic Q4) | ~0.3 GB | +| KV cache (8k ctx, Q4_0 K/V) | ~0.5 GB | +| **Total** | **~6.3 GB** | + +--- + +## Qwen3 thinking mode + +The model emits `<|think|>...<|/think|>` before its answer. +`example.cpp` strips this with `strip_think()` before printing. +The think block improves RAG quality — the model explicitly reasons +over injected context chunks before answering. + +To expose reasoning (useful for debugging retrieval quality), remove +the `strip_think()` call and print `raw` directly. diff --git a/llama/README.md b/llama/README.md new file mode 100644 index 0000000..29f99c8 --- /dev/null +++ b/llama/README.md @@ -0,0 +1,321 @@ +# SmallBASIC Llama Module + +A comprehensive SmallBASIC library module that bridges the scripting capabilities of SmallBASIC with the power of Llama.cpp Large Language Models. This project allows developers to create, configure, and interact with LLM instances directly within a SmallBASIC environment. + +## Table of Contents +1. [System Requirements & CUDA Setup](#system-requirements--cuda-setup) +2. [Obtaining Models from Hugging Face](#obtaining-models-from-hugging-face) +3. [Architecture](#architecture) +4. [Features](#features) +5. [Usage Examples](#usage-examples) +6. [API Reference](#api-reference) +7. [Configuration Presets](#configuration-presets) + +--- + +## System Requirements & CUDA Setup + +For optimal performance, especially on NVIDIA hardware, the CUDA toolkit must be correctly configured. + +### 1. Check NVIDIA Drivers +Ensure the NVIDIA open driver is installed and working: +```bash +nvidia-smi +``` +If this command works, the proprietary driver is not strictly necessary for CUDA toolkit installation. + +### 2. Add NVIDIA CUDA Repository +For Debian 12: +```bash +wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb +sudo apt update +``` + +### 3. Install CUDA Toolkit +Install only the toolkit (no driver replacement): +```bash +sudo apt install -y cuda-toolkit +``` +This installs `nvcc`, headers, and runtime libraries. + +### 4. Environment Variables +Add the following to your environment: +```bash +export PATH=/usr/local/cuda/bin:$PATH +export CUDAToolkit_ROOT=/usr/local/cuda +``` +To make this permanent, add to `~/.bashrc` and source it. + +### 5. Verify Installation +```bash +nvcc --version +``` +Output should indicate the release version (e.g., release 12.4). + +### 6. Build Configuration +When building the module, ensure the build directory is clean and configured for the CUDA backend: +```bash +rm -rf build +mkdir build +cd build +cmake -DLLAMA_BACKEND=CUDA .. +make -j$(nproc) +``` +*Note: Fully static builds are not possible for CUDA; some `.so` libraries will remain dynamically linked.* + +--- + +## Obtaining Models from Hugging Face + +The `LLAMA` function expects a path to a model file (e.g., `gguf` format). Models can be obtained from the Hugging Face Hub. + +### Method 1: Using `huggingface-cli` (Recommended) + +1. **Setup Environment** + Create a virtual environment (optional but recommended) and install the CLI tool: + ```bash + pyenv virtualenv 3.10.13 hf-tools + pyenv activate hf-tools + pip install -U pip + pip install huggingface_hub + ``` + +2. **Login** + Authenticate with your Hugging Face account: + ```bash + huggingface-cli login + ``` + (Follow the prompts to enter your token). + +3. **Download Model** + Use the `huggingface-cli download` command to fetch the model directly to your desired directory. + ```bash + # Example: Download Llama-3-8B-Instruct + huggingface-cli download meta-llama/Meta-Llama-3-8B-Instruct --include "*.gguf" --local-dir models/llama3-8b + ``` + + *Note: This command downloads all `.gguf` files associated with the repository into the `models/llama3-8b` folder.* + +### Method 2: Using Python (`huggingface_hub`) + +If you prefer a scriptable approach: +```python +from huggingface_hub import hf_hub_download + +model_path = hf_hub_download( + repo_id="meta-llama/Meta-Llama-3-8B-Instruct", + filename="llama-3-8b-instruct.Q4_K_M.gguf", # Specify exact file if needed + local_dir="models", + local_dir_use_symlinks=False +) +``` + +Once the model file is in your `models` directory (or wherever specified), you can reference it in SmallBASIC: +```basic +llama = LLAMA("models/llama3-8b/llama-3-8b-instruct.Q4_K_M.gguf", 2048, 1024, -1, 0) +``` + +### Method 3: Direct download + +1. Navigate to https://huggingface.co/ +2. Click Models at the top and then select Libraries/GGUF +3. Use the parameters slider to limit the selection for your hardware. + +--- + +## Architecture + +The module operates as a compiled library (`SBLIB`) exposing C++ functionality to SmallBASIC scripts. + +### Core Components +1. **Llama Instance Manager (`g_llama`)**: + * Stores active Llama models in a hash map keyed by ID. + * Supports initialization with custom context sizes, batch sizes, and GPU acceleration. + * Handles memory cleanup to prevent leaks. + +2. **Response Iterator (`g_llama_iter`)**: + * Manages the streaming response of an LLM. + * Provides token-by-token access to generated text. + * Tracks generation speed (`tokens/sec`) and remaining tokens. + +3. **Command Interface**: + * Exposes a set of SmallBASIC functions (callbacks) for configuration and interaction. + +--- + +## Features + +### Initialization +The `LLAMA` function creates a new model instance. +```basic +' Syntax: LLAMA(model_path, n_ctx, n_batch, n_gpu_layers, n_log_level) +' Example: +' llama = LLAMA("models/llama-7b.gguf", 2048, 1024, -1, 0) +``` + +### Configuration +Once an instance is created, various parameters can be adjusted dynamically: + +* **Temperature**: Controls randomness in generation. +* **Top-K / Top-P**: Nucleus sampling parameters. +* **Max Tokens**: Limits the length of the response. +* **Penalties**: Frequency, presence, and repeat penalties to avoid repetition. +* **Grammar**: Constrains output to specific patterns. + +```basic +' Examples: +llama.set_temperature(0.8) +llama.set_max_tokens(50) +llama.set_penalty_repeat(0.8) +llama.set_seed(123) +``` + +### Interaction +The primary method of interaction is `add_message`, which sends a prompt to the model. + +```basic +' Syntax: llama.add_message(role, content) +' Returns: An iterator object for the response. +response = llama.add_message("user", "Please describe a sunset in poetry.") +``` + +### Streaming Responses +The returned iterator allows real-time processing of the model's output: + +* `response.all()`: Returns the complete generated text. +* `response.next()`: Retrieves the next token. +* `response.has_next()`: Checks if more tokens are available. +* `response.tokens_sec`: Calculates current generation speed. + +```basic +' Example loop: +while response.has_next() + print response.next() + sleep 100 +end while +``` + +--- + +## Usage Examples + +### Factual Answers & Tool Use +*Best for: Summaries, code generation, technical queries.* +```basic +llama.set_max_tokens(150) +llama.set_temperature(0.0) +llama.set_top_k(1) +llama.set_top_p(0.0) +llama.set_min_p(0.0) +``` + +### Assistant / Q&A / Chat +*Best for: Conversational agents, explanations.* +```basic +llama.set_max_tokens(150) +llama.set_temperature(0.8) +llama.set_top_k(40) +llama.set_top_p(0.0) +llama.set_min_p(0.05) +``` + +### Creative Writing & Storytelling +*Best for: Fiction, poetry, imaginative tasks.* +```basic +llama.set_max_tokens(200) +llama.set_temperature(1.0) +llama.set_top_k(80) +llama.set_top_p(0.0) +llama.set_min_p(0.1) +``` + +### Technical & Conservative +*Best for: Documentation, logic, precise tasks.* +```basic +llama.set_max_tokens(150) +llama.set_temperature(0.6) +llama.set_top_k(30) +llama.set_top_p(0.0) +llama.set_min_p(0.02) +``` + +### Speed Optimized (CPU) +*Best for: Rapid iteration or low-resource environments.* +```basic +' llama.set_max_tokens(10) +' llama.set_temperature(0.7) +' llama.set_top_k(20) +' llama.set_top_p(0.0) +' llama.set_min_p(0.05) +``` + +--- + +## API Reference + +### Class: Llama +| Method | Description | +| :--- | :--- | +| `add_stop(text)` | Adds a stop sequence to the generation. | +| `set_penalty_repeat(value)` | Sets repeat penalty (default 1.1). | +| `set_penalty_freq(value)` | Sets frequency penalty. | +| `set_penalty_present(value)` | Sets presence penalty. | +| `set_penalty_last_n(value)` | Sets penalty context size. | +| `set_max_tokens(value)` | Sets maximum output tokens. | +| `set_min_p(value)` | Sets minimum probability threshold. | +| `set_temperature(value)` | Sets generation temperature. | +| `set_top_k(value)` | Sets top-k sampling. | +| `set_top_p(value)` | Sets top-p sampling. | +| `set_grammar(text)` | Sets output grammar constraint. | +| `set_seed(value)` | Sets random seed for reproducibility. | +| `reset()` | Clears the current conversation context. | +| `add_message(role, content)` | Sends a message and returns an iterator. | + +### Class: LlamaIter +| Method | Description | +| :--- | :--- | +| `all()` | Returns the full string of the response. | +| `has_next()` | Returns true if more tokens are available. | +| `next()` | Returns the next token string. | +| `tokens_sec` | Returns current tokens per second. | + +--- + +## Repetition Control Strategies + +### Conservative (Minimal Control) +*Use when occasional repetition is acceptable.* +```basic +llama.set_penalty_last_n(64) +llama.set_penalty_repeat(1.05) +``` + +### Balanced (Default) +*Recommended for general usage.* +```basic +llama.set_penalty_last_n(64) +llama.set_penalty_repeat(1.1) +``` + +### Aggressive (Strong Anti-Repetition) +*Use for long-form generation where repetition must be avoided.* +```basic +llama.set_penalty_last_n(128) +llama.set_penalty_repeat(1.2) +``` + +### Disabled +*Use when repetition is desired or irrelevant.* +```basic +llama.set_penalty_last_n(0) +llama.set_penalty_repeat(1.0) +``` + +--- + +## Conclusion + +This module empowers SmallBASIC users to build sophisticated AI applications, from chatbots to creative writing tools, leveraging the efficiency of Llama.cpp within a familiar scripting paradigm. Proper configuration of CUDA and generation parameters ensures optimal performance and output quality. Models can be easily acquired via the Hugging Face Hub using standard CLI tools or Python scripts. + +--- diff --git a/llama/llama-sb-rag.cpp b/llama/llama-sb-rag.cpp new file mode 100644 index 0000000..0b11d04 --- /dev/null +++ b/llama/llama-sb-rag.cpp @@ -0,0 +1,445 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#include "llama-sb.h" +#include "llama-sb-rag.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static constexpr uint32_t MAGIC = 0x52414744; +static constexpr size_t MIN_CHUNK = 40; +static constexpr const char *INSTRUCT_EMBED = "Instruct: Represent this API documentation for code retrieval\nQuery: "; +static constexpr const char *INSTRUCT_QUERY = "Instruct: Given a programming question, retrieve relevant API documentation\nQuery: "; + +enum class ChunkType { + Function, Struct, Enum, Typedef, Defines, Other +}; + +static std::string type_name(ChunkType t) { + switch (t) { + case ChunkType::Function: return "function"; + case ChunkType::Struct: return "struct"; + case ChunkType::Enum: return "enum"; + case ChunkType::Typedef: return "typedef"; + case ChunkType::Defines: return "defines"; + default: return "other"; + } +} + +/* ── helpers ───────────────────────────────────────────────── */ + +static bool starts_with(const std::string &s, const std::string &prefix) { + return s.size() >= prefix.size() && + s.compare(0, prefix.size(), prefix) == 0; +} + +static bool is_blank(const std::string &s) { + for (char c : s) if (!isspace((unsigned char)c)) return false; + return true; +} + +/* ── state machine ─────────────────────────────────────────── */ + +enum class State { + Idle, BlockComment, LineComment, Declaration, Struct, Defines +}; + +template + +static bool chunk_file(const fs::path &path, EmitChunk emit_chunk) { + std::ifstream f(path); + if (!f) { + return false; + } + + const std::string source = path.filename().string(); + + State state = State::Idle; + std::string chunk; + ChunkType chunk_type = ChunkType::Other; + int brace_depth = 0; + int paren_depth = 0; + int define_count = 0; + + auto flush = [&](ChunkType t) { + emit_chunk(source, t, chunk); + chunk.clear(); + state = State::Idle; + brace_depth = 0; + paren_depth = 0; + }; + + std::string line; + while (std::getline(f, line)) { + /* trim trailing CR */ + if (!line.empty() && line.back() == '\r') line.pop_back(); + + /* find first non-whitespace for prefix checks */ + size_t trim_pos = 0; + while (trim_pos < line.size() && + (line[trim_pos] == ' ' || line[trim_pos] == '\t')) ++trim_pos; + const std::string trimmed = line.substr(trim_pos); + + /* ── #define handling ─────────────────────────────────── */ + if (starts_with(trimmed, "#define ")) { + if (state == State::BlockComment || state == State::LineComment) { + chunk += line + "\n"; + state = State::Defines; + define_count = 1; + } else if (state == State::Defines) { + chunk += line + "\n"; + define_count++; + } else { + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + chunk.clear(); + chunk += line + "\n"; + state = State::Defines; + define_count = 1; + } + continue; + } + + /* non-define while in define group */ + if (state == State::Defines) { + flush(ChunkType::Defines); + define_count = 0; + /* fall through to process this line normally */ + } + + /* ── block comment start ──────────────────────────────── */ + if ((starts_with(trimmed, "/*") || starts_with(trimmed, "/**")) && + state == State::Idle) { + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + chunk.clear(); + chunk_type = ChunkType::Other; + chunk += line + "\n"; + state = (trimmed.find("*/", 2) != std::string::npos) + ? State::LineComment + : State::BlockComment; + continue; + } + + /* ── inside block comment ─────────────────────────────── */ + if (state == State::BlockComment) { + chunk += line + "\n"; + if (trimmed.find("*/") != std::string::npos) + state = State::LineComment; + continue; + } + + /* ── // line comment ──────────────────────────────────── */ + if (starts_with(trimmed, "//")) { + if (state == State::Idle) { + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + chunk.clear(); + chunk += line + "\n"; + state = State::LineComment; + } else if (state == State::LineComment) { + chunk += line + "\n"; + } + continue; + } + + /* ── blank line ───────────────────────────────────────── */ + if (is_blank(trimmed)) { + if (state == State::LineComment) + flush(ChunkType::Other); + else if (state == State::Idle && chunk.size() >= MIN_CHUNK) + flush(chunk_type); + continue; + } + + /* ── skip preprocessor noise ──────────────────────────── */ + if (starts_with(trimmed, "#ifndef") || starts_with(trimmed, "#ifdef") || + starts_with(trimmed, "#endif") || starts_with(trimmed, "#pragma") || + starts_with(trimmed, "#include")) { + if (state == State::LineComment || state == State::BlockComment) { + chunk.clear(); + state = State::Idle; + } + continue; + } + + /* ── typedef struct / enum start ─────────────────────── */ + if ((starts_with(trimmed, "typedef struct") || + starts_with(trimmed, "typedef enum") || + starts_with(trimmed, "struct ") || + starts_with(trimmed, "enum ")) && + (state == State::Idle || state == State::LineComment)) { + + if (state == State::Idle && chunk.size() >= MIN_CHUNK) + emit_chunk(source, chunk_type, chunk); + + /* preserve any comment already in chunk */ + if (state == State::Idle) chunk.clear(); + + chunk += line + "\n"; + chunk_type = starts_with(trimmed, "typedef") ? ChunkType::Typedef + : starts_with(trimmed, "enum ") ? ChunkType::Enum + : ChunkType::Struct; + state = State::Struct; + for (char c : line) { + if (c == '{') ++brace_depth; + if (c == '}') --brace_depth; + } + if (brace_depth <= 0 && line.find(';') != std::string::npos) + flush(chunk_type); + continue; + } + + /* ── inside struct/enum body ──────────────────────────── */ + if (state == State::Struct) { + chunk += line + "\n"; + for (char c : line) { + if (c == '{') ++brace_depth; + if (c == '}') --brace_depth; + } + if (brace_depth <= 0 && line.find(';') != std::string::npos) + flush(chunk_type); + continue; + } + + /* ── function / other declaration ────────────────────── */ + if (state == State::LineComment || state == State::Idle) { + if (state == State::Idle && chunk.size() >= MIN_CHUNK) { + emit_chunk(source, chunk_type, chunk); + chunk.clear(); + } + chunk += line + "\n"; + chunk_type = ChunkType::Function; + state = State::Declaration; + for (char c : line) { + if (c == '(') ++paren_depth; + if (c == ')') --paren_depth; + } + if (paren_depth <= 0 && line.find(';') != std::string::npos) + flush(ChunkType::Function); + continue; + } + + /* ── multi-line declaration ───────────────────────────── */ + if (state == State::Declaration) { + chunk += line + "\n"; + for (char c : line) { + if (c == '(') ++paren_depth; + if (c == ')') --paren_depth; + } + if (paren_depth <= 0 && line.find(';') != std::string::npos) + flush(ChunkType::Function); + continue; + } + } + + /* flush remainder */ + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + + return true; +} + +// +// cosine similarity (vectors already L2-normalized) +// +static float rag_cosine(const std::vector &a, + const std::vector &b) { + float dot = 0.0f; + size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; i++) { + dot += a[i] * b[i]; + } + return dot; +} + +// +// build context string from ranked results +// +static std::string rag_build_context(const RagDB &db, + const std::vector &indices, + const std::vector &scores) { + std::ostringstream out; + for (size_t i = 0; i < indices.size(); i++) { + const RagChunk &c = db.chunks[indices[i]]; + out << "// source: " << c.source + << " [" << c.type << "]" + << " (score: " << scores[i] << ")\n" + << c.text << "\n---\n"; + } + return out.str(); +} + +// +// index the file +// +bool Llama::rag_index(RagDB &db, const std::string &filepath) { + bool embed_fail = false; + auto emit_chunk = [&](const std::string &source, ChunkType type, + const std::string &text) { + if (text.size() > MIN_CHUNK) { + RagChunk chunk; + chunk.text = text; + chunk.source = source; + chunk.type = type_name(type); + if (!embed_text(INSTRUCT_EMBED + text, chunk.embedding, db.embed_dim)) { + embed_fail = true; + } else { + db.chunks.push_back(std::move(chunk)); + } + } + }; + + return !embed_fail && chunk_file(filepath, emit_chunk); +} + +// +// retrieve with session +// +std::string Llama::rag_retrieve(const RagDB &db, + const std::string &query, + int top_k, + RagSession &session) { + if (db.empty()) { + _last_error = "no input"; + return {}; + } + + std::vector qvec; + std::string text = INSTRUCT_QUERY + query; + if (!embed_text(text, qvec, db.embed_dim)) { + _last_error = "failed to embed text"; + return {}; + } + + // score all chunks + std::vector order(db.size()); + std::iota(order.begin(), order.end(), 0); + std::vector scores(db.size()); + for (int i = 0; i < db.size(); i++) { + scores[i] = rag_cosine(qvec, db.chunks[i].embedding); + } + std::sort(order.begin(), order.end(), [&](int a, int b){ return scores[a] > scores[b]; }); + + // collect top_k unseen, within budget, above threshold + std::vector result_idx; + std::vector result_scores; + + for (int idx : order) { + if ((int)result_idx.size() >= top_k) break; + if (session.is_seen(idx)) continue; + if (scores[idx] < session.score_threshold) break; /* sorted, so stop */ + if (!session.budget_ok(db.chunks[idx].text)) break; + + result_idx.push_back(idx); + result_scores.push_back(scores[idx]); + session.mark(idx); + session.charge(db.chunks[idx].text); + } + + return rag_build_context(db, result_idx, result_scores); +} + +bool RagDB::save(const std::string &path) { + std::ofstream f(path, std::ios::binary); + if (!f) { + return false; + } + + auto write32 = [&](uint32_t v) { f.write((char*)&v, 4); }; + auto write16 = [&](uint16_t v) { f.write((char*)&v, 2); }; + auto write8 = [&](uint8_t v) { f.write((char*)&v, 1); }; + auto writestr = [&](const std::string &s, size_t max_len) { + size_t len = std::min(s.size(), max_len); + f.write(s.c_str(), (std::streamsize)len); + }; + + write32(MAGIC); /* magic "RAGD" */ + write32(2); /* version */ + write32((uint32_t)chunks.size()); /* n_chunks */ + write32((uint32_t)embed_dim); /* embed_dim */ + + for (const RagChunk &c : chunks) { + write32((uint32_t)c.text.size()); + f.write(c.text.c_str(), (std::streamsize)c.text.size()); + + uint16_t src_len = (uint16_t)std::min(c.source.size(), (size_t)65535); + write16(src_len); + writestr(c.source, src_len); + + uint8_t type_len = (uint8_t)std::min(c.type.size(), (size_t)255); + write8(type_len); + writestr(c.type, type_len); + + f.write((char*)c.embedding.data(), + (std::streamsize)(embed_dim * sizeof(float))); + } + + return f.good(); +} + +bool RagDB::load(const std::string &path) { + std::ifstream f(path, std::ios::binary); + if (!f) { + return false; + } + + auto read32 = [&]() -> uint32_t { + uint32_t v = 0; f.read((char*)&v, 4); return v; + }; + auto read16 = [&]() -> uint16_t { + uint16_t v = 0; f.read((char*)&v, 2); return v; + }; + auto read8 = [&]() -> uint8_t { + uint8_t v = 0; f.read((char*)&v, 1); return v; + }; + auto readstr = [&](size_t len) -> std::string { + std::string s(len, '\0'); + f.read(&s[0], (std::streamsize)len); + return s; + }; + + uint32_t magic = read32(); + uint32_t version = read32(); + uint32_t n = read32(); + uint32_t edim = read32(); + + if (magic != MAGIC) { + return false; + } + if (version != 2) { + return false; + } + + embed_dim = (int)edim; + chunks.resize(n); + + for (uint32_t i = 0; i < n; i++) { + RagChunk &c = chunks[i]; + + uint32_t text_len = read32(); + c.text = readstr(text_len); + + uint16_t src_len = read16(); + c.source = readstr(src_len); + + uint8_t type_len = read8(); + c.type = readstr(type_len); + + c.embedding.resize(edim); + f.read((char*)c.embedding.data(), (std::streamsize)(edim * sizeof(float))); + } + + return true; +} diff --git a/llama/llama-sb-rag.h b/llama/llama-sb-rag.h new file mode 100644 index 0000000..0296f26 --- /dev/null +++ b/llama/llama-sb-rag.h @@ -0,0 +1,78 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#pragma once + +struct RagChunk { + std::string text; + std::string source; + std::string type; + std::vector embedding; +}; + +/* ── on-disk chunk (variable-length text) ──────────────────── */ +/* + * db header (16 bytes): + * uint32 magic = 0x52414744 "RAGD" + * uint32 version = 2 + * uint32 n_chunks + * uint32 embed_dim + * + * per chunk: + * uint32 text_len + * char[] text (text_len bytes, no null) + * uint16 source_len + * char[] source (source_len bytes, no null) + * uint8 type_len + * char[] type (type_len bytes, no null) + * float[] embedding (embed_dim floats) + */ +struct RagDB { + std::vector chunks; + int embed_dim = 0; + + bool load(const std::string &path); + bool save(const std::string &path); + + int size() const { return (int)chunks.size(); } + bool empty() const { return chunks.empty(); } +}; + +// +// per-session deduplication + token budget +// +struct RagSession { + std::vector seen; /* sized to db.size() on init */ + int tokens_used = 0; + int tokens_max = 0; /* set to your n_ctx */ + float score_threshold = 0.60f; /* skip weak matches */ + + void init(int n_chunks, int ctx_size) { + seen.assign(n_chunks, false); + tokens_used = 0; + tokens_max = ctx_size; + } + + void reset() { + std::fill(seen.begin(), seen.end(), false); + tokens_used = 0; + } + + bool is_seen(int idx) const { return idx < (int)seen.size() && seen[idx]; } + void mark(int idx) { if (idx < (int)seen.size()) seen[idx] = true; } + + /* rough token estimate: 1 token ≈ 4 chars */ + bool budget_ok(const std::string &text) const { + return tokens_max == 0 || + (tokens_used + (int)text.size() / 4) < (int)(tokens_max * 0.85f); + } + + void charge(const std::string &text) { + tokens_used += (int)text.size() / 4; + } +}; + diff --git a/llama/llama-sb.cpp b/llama/llama-sb.cpp new file mode 100644 index 0000000..e1a07d5 --- /dev/null +++ b/llama/llama-sb.cpp @@ -0,0 +1,738 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#include +#include +#include +#include +#include "ggml-cuda.h" + +#include "llama.h" +#include "llama-sb.h" + +constexpr int MAX_REPEAT = 50; + +static bool read_vram(size_t &used, size_t &total) { + size_t free = 0; + total = 0; +#ifdef GGML_USE_CUDA + ggml_backend_cuda_get_device_memory(0, &free, &total); + if (total > 0) { + used = total - free; + return true; + } +#endif + return false; +} + +LlamaIter::LlamaIter() : + _llama(nullptr), + _repetition_count(0), + _tokens_generated(0), + _has_next(false) { +} + +LlamaIter::LlamaIter(LlamaIter &&other) noexcept + : _llama(std::exchange(other._llama, nullptr)) + , _last_word(std::move(other._last_word)) + , _t_start(std::move(other._t_start)) + , _repetition_count(other._repetition_count) + , _tokens_generated(other._tokens_generated) + , _has_next(other._has_next) { +} + +Llama::Llama() : + _model(nullptr), + _ctx(nullptr), + _sampler(nullptr), + _vocab(nullptr), + _penalty_last_n(0), + _penalty_repeat(0), + _penalty_freq(0.0f), + _penalty_present(0.0f), + _temperature(0), + _top_p(0), + _min_p(0), + _top_k(0), + _max_tokens(0), + _log_level(GGML_LOG_LEVEL_CONT), + _n_gpu_layers(0), + _n_system_tokens(0), + _is_gemma4(false), + _sampler_dirty(false), + _can_shift(false), + _memory_flush(false), + _seed(LLAMA_DEFAULT_SEED) { + llama_log_set([](enum ggml_log_level level, const char *text, void *user_data) { + Llama *llama = (Llama *)user_data; + if (level == GGML_LOG_LEVEL_ERROR && llama->_last_error.empty()) { + // remember the first error message + llama->_last_error = text; + } + if (level > llama->_log_level) { + fprintf(stderr, "LLAMA: %s", text); + } + }, this); + reset(); + llama_backend_init(); +} + +Llama::Llama(Llama &&other) noexcept + : _model(std::exchange(other._model, nullptr)) + , _ctx(std::exchange(other._ctx, nullptr)) + , _sampler(std::exchange(other._sampler, nullptr)) + , _vocab(std::exchange(other._vocab, nullptr)) + , _stop_sequences(std::move(other._stop_sequences)) + , _grammar_src(std::move(other._grammar_src)) + , _grammar_root(std::move(other._grammar_root)) + , _last_error(std::move(other._last_error)) + , _template(std::move(other._template)) + , _penalty_last_n(other._penalty_last_n) + , _penalty_repeat(other._penalty_repeat) + , _penalty_freq(other._penalty_freq) + , _penalty_present(other._penalty_present) + , _temperature(other._temperature) + , _top_p(other._top_p) + , _min_p(other._min_p) + , _top_k(other._top_k) + , _max_tokens(other._max_tokens) + , _log_level(other._log_level) + , _n_gpu_layers(other._n_gpu_layers) + , _n_system_tokens(other._n_system_tokens) + , _is_gemma4(other._is_gemma4) + , _sampler_dirty(other._sampler_dirty) + , _can_shift(other._can_shift) + , _memory_flush(other._memory_flush) + , _seed(other._seed) { +} + +Llama::~Llama() { + if (_sampler) { + llama_sampler_free(_sampler); + } + if (_ctx) { + llama_free(_ctx); + } + if (_model) { + llama_model_free(_model); + } + llama_backend_free(); +} + +void Llama::reset() { + _stop_sequences.clear(); + _last_error.clear(); + _penalty_last_n = 64; + _penalty_repeat = 1.1f; + _penalty_freq = 0.0f; + _penalty_present = 0.0f; + _temperature = 0; + _top_k = 0; + _top_p = 1.0f; + _min_p = 0.0f; + _max_tokens = 150; + _n_system_tokens = 0; + _seed = LLAMA_DEFAULT_SEED; + _sampler_dirty = true; + if (_ctx) { + llama_memory_clear(llama_get_memory(_ctx), true); + } +} + +bool Llama::is_memory_flush() { + auto result = _memory_flush; + if (result) { + _memory_flush = false; + } + return result; +} + +bool Llama::load_model(string model_path, int n_ctx, int n_batch, int n_gpu_layers, int log_level) { + ggml_backend_load_all(); + + llama_model_params mparams = llama_model_default_params(); + if (n_gpu_layers >= 0) { + mparams.n_gpu_layers = n_gpu_layers; + } + + _last_error.clear(); + _log_level = log_level; + _n_gpu_layers = n_gpu_layers; + _model = llama_model_load_from_file(model_path.c_str(), mparams); + if (!_model) { + set_last_error("Load model"); + } else { + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = n_ctx; + cparams.n_batch = n_batch; + cparams.n_ubatch = n_batch; + cparams.no_perf = true; + cparams.attention_type = LLAMA_ATTENTION_TYPE_UNSPECIFIED; + cparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + + // or Q4_0 for more aggressive saving + cparams.type_k = GGML_TYPE_Q4_0; + cparams.type_v = GGML_TYPE_Q4_0; + + // keep KV cache on GPU + cparams.offload_kqv = true; + + _ctx = llama_init_from_model(_model, cparams); + if (!_ctx) { + set_last_error("Create context"); + } else { + _vocab = llama_model_get_vocab(_model); + _template = llama_model_chat_template(_model, nullptr); + _is_gemma4 = (_template.find("<|turn>model") != string::npos); + _can_shift = llama_memory_can_shift(llama_get_memory(_ctx)); + } + } + + return _last_error.empty(); +} + +bool Llama::load_embedding_model(string model_path) { + ggml_backend_load_all(); + + llama_model_params mparams = llama_model_default_params(); + mparams.n_gpu_layers = 99; + + _last_error.clear(); + _model = llama_model_load_from_file(model_path.c_str(), mparams); + if (!_model) { + set_last_error("Load model"); + } else { + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = 512; + cparams.n_batch = 512; + cparams.embeddings = true; + cparams.pooling_type = LLAMA_POOLING_TYPE_MEAN; + + _ctx = llama_init_from_model(_model, cparams); + if (!_ctx) { + set_last_error("Create context"); + } else { + _vocab = llama_model_get_vocab(_model); + } + } + + return _last_error.empty(); +} + +void Llama::set_grammar(const string &src, const string &root) { + _grammar_src = src; + _grammar_root = root; + dirty(); +} + +bool Llama::add_message(LlamaIter &iter, const string &role, const string &content) { + llama_chat_message message = {role.c_str(), content.c_str()}; + int buf_size = 2 * (int)(role.size() + content.size() + 64); + vector buf(buf_size); + int32_t n = 0; + + if (_template.empty()) { + set_last_error("No chat template available"); + return false; + } + + if (_is_gemma4) { + // see: https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 + string str; + if (role == "system") { + str = "<|turn>system\n<|think|>" + content + "\n"; + } else { + str = "<|turn>" + role + "\n" + content + "\n"; + } + n = str.size(); + buf.assign(str.begin(), str.end()); + buf.push_back('\0'); + } else { + bool add_ass = (role == "user" || role == "tool" || role == "tool_result"); + n = llama_chat_apply_template(_template.c_str(), &message, 1, add_ass, buf.data(), buf_size); + if (n < 0) { + set_last_error("No chat template no supported"); + return false; + } else if (n > (int32_t)buf.size()) { + buf.resize(n); + llama_chat_apply_template(_template.c_str(), &message, 1, add_ass, buf.data(), buf.size()); + } + } + string prompt(buf.data(), n); + + if (_sampler_dirty) { + // avoid wasteful rebuild + if (!configure_sampler()) { + return false; + } + _sampler_dirty = false; + } + + vector prompt_tokens = tokenize(prompt); + if (prompt_tokens.size() == 0) { + return false; + } + + if (role == "system") { + // always retain system tokens + _n_system_tokens = prompt_tokens.size(); + } + + if (!make_space_for_tokens((prompt_tokens.size() * 3) / 2)) { + return false; + } + + // batch decode tokens + if (!batch_decode_tokens(prompt_tokens)) { + return false; + } + + // handle encoder models + if (llama_model_has_encoder(_model)) { + // for example: T5, BART, and mBART. + // Used for translation, summarization, text-to-text, paraphrasing, question answering + llama_token decoder_start_token_id = llama_model_decoder_start_token(_model); + if (decoder_start_token_id == LLAMA_TOKEN_NULL) { + decoder_start_token_id = llama_vocab_bos(_vocab); + } + + llama_batch decoder_batch = llama_batch_get_one(&decoder_start_token_id, 1); + if (llama_decode(_ctx, decoder_batch)) { + set_last_error("Failed to evaluate decoder start token"); + return false; + } + } + + iter._tokens_generated = 0; + iter._t_start = std::chrono::high_resolution_clock::now(); + iter._llama = this; + iter._has_next = true; + return true; +} + +string Llama::next(LlamaIter &iter) { + if (!iter._has_next) { + set_last_error("Iteration beyond end of stream"); + return ""; + } + + // sample the next token from the current logits + llama_token tok = llama_sampler_sample(_sampler, _ctx, -1); + + // end-of-generation check + if (llama_vocab_is_eog(_vocab, tok)) { + iter._has_next = false; + return ""; + } + + string result = token_to_string(iter, tok); + + // prepare the next batch with the sampled token + llama_batch batch = llama_batch_get_one(&tok, 1); + if (llama_decode(_ctx, batch)) { + set_last_error("Failed to evaluate token during generation"); + return ""; + } + + return result; +} + +string Llama::all(LlamaIter &iter) { + string out; + + vector decoded; + decoded.reserve(_max_tokens); + + int generated = 0; + + while (generated < _max_tokens) { + // sample the next token from the current logits + llama_token tok = llama_sampler_sample(_sampler, _ctx, -1); + + // end-of-generation check + if (llama_vocab_is_eog(_vocab, tok)) { + break; + } + + // append token to decoded list + decoded.push_back(tok); + ++generated; + + // decode the token + llama_batch batch = llama_batch_get_one(&tok, 1); + if (llama_decode(_ctx, batch)) { + set_last_error("Failed to evaluate token during generation"); + break; + } + } + + // tokens exhausted - call add_message to continue + iter._has_next = false; + + // detokenize sequentially + if (!decoded.empty()) { + for (llama_token tok : decoded) { + out.append(token_to_string(iter, tok)); + } + } + + return out; +} + +float Llama::memory_kv_percent() { + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + int n_ctx = llama_n_ctx(_ctx); + int kv_used = (pos_max < 0) ? 0 : (int)pos_max + 1; + return 100.0f * kv_used / n_ctx; +} + +LlamaMemoryInfo Llama::memory_info() { + LlamaMemoryInfo info = {}; + + // KV cache usage + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + int n_ctx = llama_n_ctx(_ctx); + info.kv_total = n_ctx; + info.kv_used = (pos_max < 0) ? 0 : (int)pos_max + 1; + info.kv_percent = 100.0f * info.kv_used / info.kv_total; + + // Model layers + auto n_gpu_layers = std::max(0, _n_gpu_layers); + info.n_layers_total = llama_model_n_layer(_model); + info.n_layers_gpu = std::min(info.n_layers_total, n_gpu_layers); + info.n_layers_cpu = info.n_layers_total - info.n_layers_gpu; + + // ram + if (read_vram(info.vram_used, info.vram_total)) { + info.vram_percent = 100.0f * info.vram_used / info.vram_total; + } + + info.model_native_max_ctx = llama_model_n_ctx_train(_model); + + // Advice + ostringstream advice; + + // Check structural limits & model configuration quirks + if (info.kv_total > info.model_native_max_ctx) { + advice << "WARNING: Configured context size (" << info.kv_total + << ") exceeds model native training length (" << info.model_native_max_ctx + << "). Logic flaws or repetition bugs will occur unless RoPE scaling options are enabled. "; + } + + if (n_gpu_layers < info.n_layers_total) { + advice << "Only " << n_gpu_layers << "/" << info.n_layers_total + << " layers on GPU - increase n_gpu_layers if VRAM allows. "; + } else { + advice << "All " << info.n_layers_total << " layers on GPU. "; + } + if (info.n_layers_cpu > 0) { + advice << "CPU offload active (" << info.n_layers_cpu + << " layers on CPU) - increase n_gpu_layers if VRAM allows. "; + } + if (info.vram_percent > 90.0f) { + advice << "VRAM >90% - reduce n_ctx or use Q4_0 KV cache. "; + } else if (info.vram_percent < 60.0f && info.n_layers_cpu > 0) { + advice << "VRAM headroom available - try adding more GPU layers. "; + } + if (info.kv_percent > 80.0f) { + advice << "Context >80% full - consider calling clear_history(). "; + } + info.advice = advice.str(); + + return info; +} + +bool Llama::embed_text(const std::string &text, std::vector &out, int embed_dim) { + vector tokens = tokenize(text); + if (tokens.size() == 0) { + return false; + } + + // truncate to context window + int n_ctx = llama_n_ctx(_ctx); + int n = tokens.size(); + if (n > n_ctx) { + set_last_error(std::format("warning: chunk truncated {} -> {} tokens ", n, n_ctx)); + n = n_ctx; + tokens.resize(n); + } + + llama_memory_clear(llama_get_memory(_ctx), true); + + if (!batch_decode_tokens(tokens)) { + return false; + } + + float *emb = llama_get_embeddings_seq(_ctx, 0); + if (!emb) { + emb = llama_get_embeddings_ith(_ctx, n - 1); + } + + if (!emb) { + set_last_error("no embedding returned"); + return false; + } + + out.assign(emb, emb + embed_dim); + + /* L2 normalize */ + float norm = 0.0f; + for (float v : out) { + norm += v * v; + } + norm = std::sqrt(norm); + if (norm > 1e-9f) { + for (float &v : out) { + v /= norm; + } + } + + return true; +} + +bool Llama::batch_decode_tokens(vector &tokens) { + uint32_t n_batch = llama_n_batch(_ctx); + for (size_t i = 0; i < tokens.size(); i += n_batch) { + size_t batch_size = std::min((size_t)n_batch, tokens.size() - i); + llama_batch batch = llama_batch_get_one(tokens.data() + i, batch_size); + int result = llama_decode(_ctx, batch); + if (result == 1) { + // KV full or fragmented mid-batch - evict oldest tokens and retry + if (!make_space_for_tokens(n_batch)) { + set_decode_error(result, i, tokens.size()); + return false; + } + result = llama_decode(_ctx, batch); + if (result == 1) { + // Eviction reported enough logical space but decode still failed - + // this is fragmentation, not a real space shortage. No defrag API + // is available, so fall back to a full non-system flush, which + // guarantees one contiguous block. + if (!full_flush_except_system()) { + set_decode_error(result, i, tokens.size()); + return false; + } + _memory_flush = true; + result = llama_decode(_ctx, batch); + } + } + if (result != 0) { + set_decode_error(result, i, tokens.size()); + return false; + } + } + return true; +} + +bool Llama::configure_sampler() { + auto sparams = llama_sampler_chain_default_params(); + sparams.no_perf = false; + llama_sampler *chain = llama_sampler_chain_init(sparams); + + if (!_grammar_src.empty()) { + llama_sampler *grammar = llama_sampler_init_grammar(_vocab, _grammar_src.c_str(), _grammar_root.c_str()); + if (!grammar) { + set_last_error("failed to initialize grammar sampler"); + return false; + } + llama_sampler_chain_add(chain, grammar); + } + if (_penalty_last_n != 0 && _penalty_repeat != 1.0f) { + auto penalties = llama_sampler_init_penalties(_penalty_last_n, _penalty_repeat, _penalty_freq, _penalty_present); + llama_sampler_chain_add(chain, penalties); + } + if (_temperature <= 0.0f) { + llama_sampler_chain_add(chain, llama_sampler_init_greedy()); + } else { + if (_top_k > 0) { + llama_sampler_chain_add(chain, llama_sampler_init_top_k(_top_k)); + } + if (_top_p < 1.0f || _min_p > 0.0f) { + llama_sampler_chain_add(chain, llama_sampler_init_top_p(_top_p, 1)); + } + if (_min_p > 0.0f) { + llama_sampler_chain_add(chain, llama_sampler_init_min_p(_min_p, 1)); + } + llama_sampler_chain_add(chain, llama_sampler_init_temp(_temperature)); + llama_sampler_chain_add(chain, llama_sampler_init_dist(_seed)); + } + if (_sampler) { + llama_sampler_free(_sampler); + } + _sampler = chain; + return true; +} + +bool Llama::full_flush_except_system() { + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); + if (pos_min < 0) { + return true; // already empty + } + llama_pos flush_start = pos_min + _n_system_tokens; + bool ok = llama_memory_seq_rm(mem, 0, flush_start, -1); + if (!ok) { + set_last_error("Failed to flush memory past system tokens"); + return false; + } + return true; +} + +// Makes space in the context for n_tokens by removing old tokens if necessary +// Returns true if successful, false if impossible to make space +// +// Strategies: +// - If enough space exists, does nothing +// - If n_tokens > n_ctx, fails (impossible to fit) +// - Otherwise, removes oldest tokens to make room +// +// Parameters: +// n_tokens - Number of tokens we need space for +// +bool Llama::make_space_for_tokens(int n_tokens) { + int n_ctx = llama_n_ctx(_ctx); + if (n_tokens > n_ctx) { + set_last_error("Too many tokens, increase context size (n_ctx)"); + return false; + } + + llama_memory_t mem = llama_get_memory(_ctx); + + // Get current position range + llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + + // Empty memory - nothing to do + if (pos_max < 0) { + return true; + } + + int current_used = pos_max - pos_min + 1; + int space_needed = n_tokens; + int space_available = n_ctx - current_used; + + // Already have enough space + if (space_available >= space_needed) { + return true; + } + + // Calculate how many tokens to remove + int tokens_to_remove = space_needed - space_available; + + // Can't remove more than we have (minus _n_system_tokens) + int removable = current_used - _n_system_tokens; + if (tokens_to_remove > removable) { + set_last_error("Can't make enough space while keeping num_system_tokens tokens"); + return false; + } + if (!_can_shift) { + set_last_error("Memory type doesn't support shifting, can't evict mid-sequence"); + return false; + } + + llama_pos remove_start = pos_min + _n_system_tokens; + + // Remove oldest tokens (from pos_min to pos_min + tokens_to_remove) + llama_memory_seq_rm(mem, 0, remove_start, remove_start + tokens_to_remove); + + // Shift remaining tokens down + llama_memory_seq_add(mem, 0, remove_start + tokens_to_remove, -1, -tokens_to_remove); + + set_last_error(std::format("made space for {} tokens", n_tokens)); + return true; +} + +vector Llama::tokenize(const string &prompt) { + vector result; + + int n_prompt = -llama_tokenize(_vocab, prompt.c_str(), prompt.size(), nullptr, 0, true, true); + if (n_prompt <= 0) { + set_last_error("Failed to tokenize prompt"); + } else { + result.reserve(n_prompt); + result.resize(n_prompt); + if (llama_tokenize(_vocab, prompt.c_str(), prompt.size(), + result.data(), n_prompt, true, true) < 0) { + set_last_error("Failed to tokenize prompt"); + } + } + return result; +} + +string Llama::token_to_string(LlamaIter &iter, llama_token tok) { + string result; + char buf[512]; + int n = llama_token_to_piece(_vocab, tok, buf, sizeof(buf), 0, false); + if (n > 0) { + // detect repetition - only on non-whitespace tokens, otherwise + // spaces/newlines trigger false positives almost immediately. + string piece(buf, n); + bool is_trivial = piece.find_first_not_of(" \t\n\r") == string::npos; + if (!is_trivial) { + if (iter._last_word == piece) { + if (++iter._repetition_count >= MAX_REPEAT) { + iter._has_next = false; + } + } else { + iter._repetition_count = 0; + iter._last_word = piece; + } + } + + result.append(buf, n); + + // detect end of max-tokens + if (++iter._tokens_generated > _max_tokens) { + iter._has_next = false; + } + + // detect stop words + if (iter._has_next) { + for (const auto &stop : _stop_sequences) { + size_t pos = result.find(stop); + if (pos != std::string::npos) { + // found stop sequence - truncate and signal end + result = result.substr(0, pos); + iter._has_next = false; + break; + } + } + } + } + return result; +} + +void Llama::set_last_error(const string &message) { + if (!_last_error.empty()) { + if (_last_error.back() == '\n') { + _last_error.pop_back(); + } + _last_error = std::format("{}: {}", message, _last_error); + } else { + _last_error = std::format("{} failed", message); + } +} + +void Llama::set_decode_error(int32_t error, int index, int num_tokens) { + if (error == 1) { + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + int n_ctx = llama_n_ctx(_ctx); + int current_used = pos_max - pos_min + 1; + int space_needed = num_tokens; + int space_available = n_ctx - current_used; + _n_system_tokens; + set_last_error(std::format("KV exhausted. Reduce batch or context sizes. batchNo:{} requested:{} available:{}", + index, space_needed, space_available)); + } else { + auto message = error == 2 ? "abort" : error == -1 ? "invalid" : "fatal"; + set_last_error(std::format("Failed to decode batch. batchNo:{} error:'{}'", index, message)); + } +} diff --git a/llama/llama-sb.h b/llama/llama-sb.h new file mode 100644 index 0000000..8306e19 --- /dev/null +++ b/llama/llama-sb.h @@ -0,0 +1,156 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#pragma once + +#include +#include +#include +#include "llama.h" + +using namespace std; + +struct Llama; +struct RagDB; +struct RagSession; + +struct LlamaMemoryInfo { + // KV cache + int kv_used; // slots currently used + int kv_total; // total slots (== n_ctx) + float kv_percent; // kv_used / kv_total + + // GPU VRAM (via ggml backend) + size_t vram_used; // bytes + size_t vram_total; // bytes + float vram_percent; + + // Model layers + int n_layers_total; // total model layers + int n_layers_gpu; // layers offloaded to GPU + int n_layers_cpu; // layers on CPU + int model_native_max_ctx; + + // Advice + string advice; +}; + +struct LlamaIter { + explicit LlamaIter(); + ~LlamaIter() {} + + // move constructor + LlamaIter(LlamaIter &&other) noexcept; + + // delete the copy + LlamaIter(const LlamaIter &) = delete; + LlamaIter &operator=(const LlamaIter &) = delete; + + Llama *_llama; + string _last_word; + chrono::high_resolution_clock::time_point _t_start; + int _repetition_count; + int _tokens_generated; + bool _has_next; +}; + +struct Llama { + explicit Llama(); + + // move constructor + Llama(Llama &&other) noexcept; + + // delete the copy + Llama(const Llama &) = delete; + Llama &operator=(const Llama &) = delete; + + ~Llama(); + + // init + bool load_model(string model_path, int n_ctx, int n_batch, int n_gpu_layers, int log_level); + bool load_embedding_model(string model_path); + + // generation + bool add_message(LlamaIter &iter, const string &role, const string &content); + string next(LlamaIter &iter); + string all(LlamaIter &iter); + + // generation parameters + void add_stop(const char *stop) { _stop_sequences.push_back(stop); } + void clear_stops() { _stop_sequences.clear(); } + void set_penalty_last_n(int32_t penalty_last_n) { _penalty_last_n = penalty_last_n; dirty(); } + void set_penalty_repeat(float penalty_repeat) { _penalty_repeat = penalty_repeat; dirty(); } + void set_penalty_freq(float penalty_freq) { _penalty_freq = penalty_freq; dirty(); } + void set_penalty_present(float penalty_present) { _penalty_present = penalty_present; dirty(); } + void set_max_tokens(int max_tokens) { _max_tokens = max_tokens; dirty(); } + void set_min_p(float min_p) { _min_p = min_p; dirty(); } + void set_temperature(float temperature) { _temperature = temperature; dirty(); } + void set_top_k(int top_k) { _top_k = top_k; dirty(); } + void set_top_p(float top_p) { _top_p = top_p; dirty(); } + void set_grammar(const string &src, const string &root); + void set_seed(unsigned int seed) { _seed = seed; dirty(); } + + // error handling + const char *last_error() { return _last_error.c_str(); } + void set_log_level(int level) { _log_level = level; } + void reset(); + bool is_memory_flush(); + + // memory info + LlamaMemoryInfo memory_info(); + float memory_kv_percent(); + + // creates an embedding vector of the given dimension for the given text + bool embed_text(const std::string &text, std::vector &out, int embed_dim); + + // retrieves rag query context informatiion from the rag database + std::string rag_retrieve(const RagDB &db, const std::string &query, int top_k, RagSession &session); + + // indexes the details from the given file + bool rag_index(RagDB &db, const std::string &filepath); + + // returns the emdedding dimension for the loaded model + int get_embed_dim() const { return _model != nullptr ? llama_model_n_embd(_model) : 0; } + + private: + bool batch_decode_tokens(vector &tokens); + bool configure_sampler(); + void dirty() {_sampler_dirty = true; } + bool full_flush_except_system(); + bool make_space_for_tokens(int n_tokens); + vector tokenize(const string &prompt); + string token_to_string(LlamaIter &iter, llama_token tok); + void set_last_error(const string &message); + void set_decode_error(int32_t error, int index, int num_tokens); + + llama_model *_model; + llama_context *_ctx; + llama_sampler *_sampler; + const llama_vocab *_vocab; + vector _stop_sequences; + string _grammar_src; + string _grammar_root; + string _last_error; + string _template; + int32_t _penalty_last_n; + float _penalty_repeat; + float _penalty_freq; + float _penalty_present; + float _temperature; + float _top_p; + float _min_p; + int _top_k; + int _max_tokens; + int _log_level; + int _n_gpu_layers; + int _n_system_tokens; + bool _is_gemma4; + bool _sampler_dirty; + bool _can_shift; + bool _memory_flush; + unsigned int _seed; +}; diff --git a/llama/llama.cpp b/llama/llama.cpp new file mode 160000 index 0000000..86b9470 --- /dev/null +++ b/llama/llama.cpp @@ -0,0 +1 @@ +Subproject commit 86b94708f22478f900b76ca02e316f4f3418faff diff --git a/llama/main.cpp b/llama/main.cpp new file mode 100644 index 0000000..b78d015 --- /dev/null +++ b/llama/main.cpp @@ -0,0 +1,597 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#include "config.h" + +#include "robin-hood-hashing/src/include/robin_hood.h" +#include "include/log.h" +#include "include/var.h" +#include "include/module.h" +#include "include/param.h" + +#include "llama-sb.h" + +#define CLASS_ID_LLAMA 1 +#define CLASS_ID_LLAMA_ITER 2 + +int g_nextId = 1; +robin_hood::unordered_map g_llama; +robin_hood::unordered_map g_llama_iter; + +static int get_llama_class_id(var_s *map, var_s *retval) { + int result = -1; + if (is_map(map)) { + int id = map->v.m.id; + if (id != -1 && g_llama.find(id) != g_llama.end()) { + result = id; + } + } + if (result == -1) { + error(retval, "Llama not found"); + } + return result; +} + +static int get_llama_iter_class_id(var_s *map, var_s *retval) { + int result = -1; + if (is_map(map)) { + int id = map->v.m.id; + if (id != -1 && g_llama_iter.find(id) != g_llama_iter.end()) { + result = id; + } + } + if (result == -1) { + error(retval, "Llama iter not found"); + } + return result; +} + +static string expand_path(const char *path) { + string result; + if (path && path[0] == '~') { + const char *home = getenv("HOME"); + if (home != nullptr) { + result.append(home); + result.append(path + 1); + } else { + result = path; + } + } else { + result = path; + } + return result; +} + +// +// llama.add_stop('xyz') +// +static int cmd_llama_add_stop(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.add_stop", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + llama.add_stop(get_param_str(argc, arg, 0, "stop")); + result = 1; + } + } + return result; +} + +// +// llama.set_penalty_repeat(0.8) +// +static int cmd_llama_set_penalty_repeat(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_penalty_repeat", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_repeat(value); + v_setreal(map_add_var(self, "penalty_repeat", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_penalty_freq(0.8) +// +static int cmd_llama_set_penalty_freq(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_penalty_freq", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_freq(value); + v_setreal(map_add_var(self, "penalty_freq", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_penalty_present(0.8) +// +static int cmd_llama_set_penalty_present(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_penalty_present", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_present(value); + v_setreal(map_add_var(self, "penalty_present", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_penalty_last_n(0.8) +// +static int cmd_llama_set_penalty_last_n(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_penalty_last_n", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_last_n(value); + v_setreal(map_add_var(self, "penalty_last_n", 0), value); + result = 1; + } + } + return result; +} + + +// +// llama.set_max_tokens(50) +// +static int cmd_llama_set_max_tokens(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_max_tokens", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_int(argc, arg, 0, 0); + llama.set_max_tokens(value); + v_setreal(map_add_var(self, "max_tokens", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_min_p(0.5) +// +static int cmd_llama_set_min_p(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_min_p", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_min_p(value); + v_setreal(map_add_var(self, "min_p", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_temperature(0.8) +// +static int cmd_llama_set_temperature(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_temperature", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_temperature(value); + v_setreal(map_add_var(self, "temperature", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_top_k(10.0) +// +static int cmd_llama_set_top_k(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_top_k", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_int(argc, arg, 0, 0); + llama.set_top_k(value); + v_setreal(map_add_var(self, "top_k", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_top_p(0) +// +static int cmd_llama_set_top_p(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_top_p", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_top_p(value); + v_setreal(map_add_var(self, "top_p", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_grammar("text") +// +static int cmd_llama_set_grammar(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_grammar", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_str(argc, arg, 0, 0); + llama.set_grammar(value, "root"); + v_setstr(map_add_var(self, "grammar", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_seed(123) +// +static int cmd_llama_set_seed(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_seed", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_seed(value); + v_setreal(map_add_var(self, "seed", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.reset() - make the model forget everything +// +static int cmd_llama_reset(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "llama.reset", 0, 0); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + llama.reset(); + result = 1; + } + } + return result; +} + +// +// iter.all() +// +static int cmd_llama_all(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "iter.all", 0, 0); + } else { + int id = get_llama_iter_class_id(self, retval); + if (id != -1) { + LlamaIter &iter = g_llama_iter.at(id); + auto out = iter._llama->all(iter); + v_setstr(retval, out.c_str()); + result = 1; + } + } + return result; +} + +// +// iter.has_next() +// +static int cmd_llama_has_next(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "iter.has_next", 0, 0); + } else { + int id = get_llama_iter_class_id(self, retval); + if (id != -1) { + LlamaIter &llamaIter = g_llama_iter.at(id); + v_setint(retval, llamaIter._has_next); + result = 1; + } + } + return result; +} + +// +// iter.next() +// +static int cmd_llama_next(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "iter.next", 0, 0); + } else { + int id = get_llama_iter_class_id(self, retval); + if (id != -1) { + LlamaIter &iter = g_llama_iter.at(id); + auto out = iter._llama->next(iter); + v_setstr(retval, out.c_str()); + result = 1; + } + } + return result; +} + +// +// iter.tokens_sec +// +static int cmd_llama_tokens_sec(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "iter.tokens_sec", 0, 0); + } else { + int id = get_llama_iter_class_id(self, retval); + if (id != -1) { + LlamaIter &iter = g_llama_iter.at(id); + auto t_end = std::chrono::high_resolution_clock::now(); + double secs = std::chrono::duration(t_end - iter._t_start).count(); + double tokens_sec = secs > 0 ? iter._tokens_generated / secs : 0; + v_setreal(retval, tokens_sec); + result = 1; + } + } + return result; +} + +// +// print llama.add_message("please generate as simple program in BASIC to draw a cat") +// +static int cmd_llama_add_message(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 2) { + error(retval, "llama.add_message", 2, 2); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + int iter_id = ++g_nextId; + LlamaIter &iter = g_llama_iter[iter_id]; + Llama &llama = g_llama.at(id); + auto role = get_param_str(argc, arg, 0, "user"); + auto content = get_param_str(argc, arg, 1, ""); + if (llama.add_message(iter, role, content)) { + map_init_id(retval, iter_id, CLASS_ID_LLAMA_ITER); + v_create_callback(retval, "all", cmd_llama_all); + v_create_callback(retval, "has_next", cmd_llama_has_next); + v_create_callback(retval, "next", cmd_llama_next); + v_create_callback(retval, "tokens_sec", cmd_llama_tokens_sec); + result = 1; + } else { + g_llama_iter.erase(iter_id); + error(retval, llama.last_error()); + } + } + } + return result; +} + +// +// print llama.mem_info() +// +static int cmd_llama_mem_info(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "llama.mem_info", 0, 0); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto mem_info = llama.memory_info(); + map_init(retval); + v_setint(map_add_var(retval, "kv_used", 0), mem_info.kv_used); + v_setint(map_add_var(retval, "kv_total", 0), mem_info.kv_total); + v_setreal(map_add_var(retval, "kv_percent", 0), mem_info.kv_percent); + v_setint(map_add_var(retval, "vram_used", 0), mem_info.vram_used); + v_setint(map_add_var(retval, "vram_total", 0), mem_info.vram_total); + v_setreal(map_add_var(retval, "vram_percent", 0), mem_info.vram_percent); + v_setint(map_add_var(retval, "n_layers_cpu", 0), mem_info.n_layers_cpu); + v_setint(map_add_var(retval, "n_layers_gpu", 0), mem_info.n_layers_gpu); + v_setint(map_add_var(retval, "n_layers_total", 0), mem_info.n_layers_total); + v_setstr(map_add_var(retval, "advice", 0), mem_info.advice.c_str()); + result = 1; + } + } + return result; +} + +static int cmd_create_llama(int argc, slib_par_t *params, var_t *retval) { + int result; + auto model = expand_path(get_param_str(argc, params, 0, "")); + auto n_ctx = get_param_int(argc, params, 1, 2048); + auto n_batch = get_param_int(argc, params, 2, 1024); + auto n_gpu_layers = get_param_int(argc, params, 3, -1); + auto n_log_level = get_param_int(argc, params, 4, GGML_LOG_LEVEL_CONT); + int id = ++g_nextId; + Llama &llama = g_llama[id]; + if (llama.load_model(model, n_ctx, n_batch, n_gpu_layers, n_log_level)) { + map_init_id(retval, id, CLASS_ID_LLAMA); + v_create_callback(retval, "add_stop", cmd_llama_add_stop); + v_create_callback(retval, "add_message", cmd_llama_add_message); + v_create_callback(retval, "reset", cmd_llama_reset); + v_create_callback(retval, "set_penalty_repeat", cmd_llama_set_penalty_repeat); + v_create_callback(retval, "set_penalty_freq", cmd_llama_set_penalty_freq); + v_create_callback(retval, "set_penalty_present", cmd_llama_set_penalty_present); + v_create_callback(retval, "set_penalty_last_n", cmd_llama_set_penalty_last_n); + v_create_callback(retval, "set_max_tokens", cmd_llama_set_max_tokens); + v_create_callback(retval, "set_min_p", cmd_llama_set_min_p); + v_create_callback(retval, "set_temperature", cmd_llama_set_temperature); + v_create_callback(retval, "set_top_k", cmd_llama_set_top_k); + v_create_callback(retval, "set_top_p", cmd_llama_set_top_p); + v_create_callback(retval, "set_grammar", cmd_llama_set_grammar); + v_create_callback(retval, "set_seed", cmd_llama_set_seed); + v_create_callback(retval, "mem_info", cmd_llama_mem_info); + result = 1; + } else { + error(retval, llama.last_error()); + g_llama.erase(id); + result = 0; + } + return result; +} + +FUNC_SIG lib_func[] = { + {1, 5, "LLAMA", cmd_create_llama}, +}; + +SBLIB_API int sblib_func_count() { + return 1; +} + +FUNC_SIG lib_proc[] = {}; + +SBLIB_API int sblib_proc_count() { + return 0; +} + +// +// Program startup +// +int sblib_init(const char *sourceFile) { + return 1; +} + +// +// Release variables falling out of scope +// +SBLIB_API int sblib_free(int cls_id, int id) { + if (id != -1) { + switch (cls_id) { + case CLASS_ID_LLAMA: + if (g_llama.find(id) != g_llama.end()) { + g_llama.erase(id); + } + break; + case CLASS_ID_LLAMA_ITER: + if (g_llama_iter.find(id) != g_llama_iter.end()) { + g_llama_iter.erase(id); + } + break; + } + } + return 0; +} + +// +// Move the mapped instance to a new position and returns the position +// +SBLIB_API int sblib_refresh_id(int cls_id, int id) { + int result = id; + if (id != -1) { + switch (cls_id) { + case CLASS_ID_LLAMA: + if (g_llama.find(id) != g_llama.end()) { + result = ++g_nextId; + auto it = g_llama.find(id); + auto value = std::move(it->second); + g_llama.erase(it); + g_llama.emplace(result, std::move(value)); + } + break; + case CLASS_ID_LLAMA_ITER: + if (g_llama_iter.find(id) != g_llama_iter.end()) { + result = ++g_nextId; + auto it = g_llama_iter.find(id); + auto value = std::move(it->second); + g_llama_iter.erase(it); + g_llama_iter.emplace(result, std::move(value)); + } + break; + } + } + return result; +} + +// +// Program termination +// +void sblib_close(void) { + if (!g_llama.empty()) { + fprintf(stderr, "LLM leak detected\n"); + g_llama.clear(); + } + if (!g_llama_iter.empty()) { + fprintf(stderr, "LLM iter leak detected\n"); + g_llama_iter.clear(); + } +} + +#if defined(ANDROID_MODULE) +// +// Retrieves the _app->activity->clazz value sent from App/JNI to Java to IOIOLoader +// +extern "C" JNIEXPORT void JNICALL Java_ioio_smallbasic_android_ModuleLoader_init + (JNIEnv *env, jclass clazz, jobject activity) { + logEntered(); + jclass longClass = env->FindClass("java/lang/Long"); + jmethodID longValueMethod = env->GetMethodID(longClass, "longValue", "()J"); + g_activity = (jobject)env->CallLongMethod(activity, longValueMethod); + g_env = env; +} + +#endif diff --git a/llama/nitro.cpp b/llama/nitro.cpp new file mode 100644 index 0000000..2f64dbd --- /dev/null +++ b/llama/nitro.cpp @@ -0,0 +1,2836 @@ +// nitro.cpp — Nitro Agent +// A standalone agentic LLM shell with notcurses TUI. +// Uses llama-sb.h as the sole llama.cpp integration layer. +// +// Usage: +// ./nitro [options] [project_dir] +// +// Options: +// -m, --model GGUF model to load on startup +// -e, --embed embedding model for RAG +// -g, --gpu-layers layers to offload to GPU (default: 32) +// +// Slash commands: +// /model — load / hot-reload a GGUF model (picker if no path) +// /embed — load an embedding model for RAG (picker if no path) +// /rag — index a file or directory into RAG +// /memory — show KV / VRAM / layer stats +// /clear — reset conversation (keeps system prompt) +// /help — list commands +// +// Tool protocol (LLM emits, Nitro executes): +// TOOL:LIST [dir] +// TOOL:READ +// TOOL:WRITE +// TOOL:EXISTS +// TOOL:RUN [args] +// TOOL:DATE +// TOOL:TIME +// TOOL:RND +// TOOL:CURL +// +// Copyright (C) 2026 Chris Warren-Smith — GPLv2 or later +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "llama-sb.h" +#include "llama-sb-rag.h" + +#include + +namespace fs = std::filesystem; + +// +// NitroConfig +// +struct NitroConfig { + std::string model_path; + std::string embed_path; + std::string sandbox; + std::string agent_id; + int n_ctx = 65536; + int n_batch = 512; + int n_gpu_layers = 32; + int log_level = GGML_LOG_LEVEL_CONT; + float temperature = 0.6f; + float top_p = 0.95f; + float min_p = 0.0f; + int top_k = 20; + float penalty_repeat = 1.0f; + int penalty_last_n = 256; + std::vector knowledge_files; + int rag_top_k = 5; + bool thinking = true; + bool permission_prompt = false; + // TOOL:RUN allowlist — if non-empty, only these program basenames may run. + // Empty means "allow anything inside the sandbox" (original behaviour). + std::vector run_allowed; +}; + +// +// InputHistory — up/down arrow navigation through submitted inputs +// +class InputHistory { + public: + explicit InputHistory() = default; + ~InputHistory() = default; + InputHistory(const InputHistory &) = delete; + InputHistory &operator=(const InputHistory &) = delete; + + /** + * @brief Adds a new command string to the history stack. + * Resets navigation index upon adding a new item. + * Deduplicates consecutive identical entries. + */ + void push(const std::string &input) { + if (input.empty()) return; + if (!history_stack.empty() && history_stack.back() == input) { + // Don't push duplicate of last entry; just reset nav position. + current_index = static_cast(history_stack.size()); + return; + } + history_stack.push_back(input); + current_index = static_cast(history_stack.size()); + } + + /** + * @brief Navigates to an earlier entry. + * @param out Set to the selected entry on success. + * @return true if an item was successfully retrieved. + */ + bool up(std::string &out) { + if (history_stack.empty() || current_index <= 0) return false; + --current_index; + out = history_stack[current_index]; + return true; + } + + /** + * @brief Navigates to a later entry, or clears when past the newest. + * @param out Set to the selected entry, or cleared if past the end. + * @return true if a history entry was retrieved (false means "clear input"). + */ + bool down(std::string &out) { + if (history_stack.empty()) return false; + ++current_index; + if (current_index >= static_cast(history_stack.size())) { + current_index = static_cast(history_stack.size()); + out.clear(); + return false; // signal: restore blank input + } + out = history_stack[current_index]; + return true; + } + + /** Reset navigation position without modifying the stack. */ + void reset_nav() { + current_index = static_cast(history_stack.size()); + } + + /** + * @brief Load history from ~/.config/nitro/nitro.history (one entry per line). + * Silently succeeds if the file doesn't exist. + */ + void load(const std::string &path) { + std::ifstream f(path); + if (!f) return; + std::string line; + while (std::getline(f, line)) { + if (!line.empty()) history_stack.push_back(line); + } + current_index = static_cast(history_stack.size()); + } + + /** + * @brief Persist history to disk (most-recent last, one entry per line). + * Caps at MAX_PERSIST entries so the file never grows unbounded. + */ + void save(const std::string &path) const { + // Ensure parent directory exists. + fs::path dir = fs::path(path).parent_path(); + std::error_code ec; + fs::create_directories(dir, ec); + + std::ofstream f(path, std::ios::trunc); + if (!f) return; + + static constexpr int MAX_PERSIST = 500; + int start = std::max(0, static_cast(history_stack.size()) - MAX_PERSIST); + for (int i = start; i < static_cast(history_stack.size()); ++i) { + // Escape embedded newlines so each entry stays on one line. + for (char c : history_stack[i]) { + if (c == '\n') f << "\\n"; + else f << c; + } + f << '\n'; + } + } + + private: + std::vector history_stack; + int current_index = 0; +}; + +// +// Notcurses TUI +// +// +// ┌──────────────────── header (1 row) ─────────────────────────────────┐ +// │ ✦ NITRO model: … tok/s: … KV: …% VRAM: …% │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ │ +// │ chat pane (rows 1 … term_rows-3) │ +// │ │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ ───────────────────────────────────── (separator) │ +// │ ❯ input │ +// └─────────────────────────────────────────────────────────────────────┘ +struct TuiState { + // ── notcurses handles ────────────────────────────────────────────── + struct notcurses *nc = nullptr; + struct ncplane *stdpl = nullptr; + struct ncplane *header = nullptr; + struct ncplane *chatpl = nullptr; + struct ncplane *inputpl = nullptr; + // ── chat buffer ─────────────────────────────────────────────────── + std::vector chat_lines; + int scroll_offset = 0; + std::mutex lines_mutex; + // ── streaming accumulator ───────────────────────────────────────── + std::string token_acc; + // ── input ───────────────────────────────────────────────────────── + std::string input_buf; + size_t cursor_pos = 0; + bool mouse_mode = true; + // ── status bar values ───────────────────────────────────────────── + std::string current_model = "none"; + float tokens_per_sec = 0.0f; + int kv_used = 0; + int kv_total = 1; + int kv_percent = 0; + size_t vram_used = 0; + size_t vram_total = 1; + int term_rows = 0; + int term_cols = 0; + // ── thinking spinner ────────────────────────────────────────────── + bool thinking = false; + int spinner_frame = 0; + // ── input history ───────────────────────────────────────────────── + InputHistory history; + // Advance spinner by one frame and redraw the header. + void tick_spinner(); + + // Toggle thinking mode; redraws header immediately. + void set_thinking(bool on); + void update_usage(int tokens_sec, const LlamaMemoryInfo &mem); + + // ── lifecycle ───────────────────────────────────────────────────── + void init(); + void destroy(); + void resize(); + // ── draw ────────────────────────────────────────────────────────── + void redraw_header() const; + void redraw_chat(); + void redraw_input() const; + void redraw_all(); + // ── content helpers ─────────────────────────────────────────────── + void append_line(const std::string &line); + void append_token(const std::string &token); + void flush_token_acc(); + // ── interaction ─────────────────────────────────────────────────── + bool confirm_dialog(const std::string &prompt) const; + // Blocking readline with history navigation, cursor, arrow-key scrolling. + std::string readline_blocking(); + // Modal popup overlay while a long operation runs. + // Call show_modal_popup to display; dismiss_modal_popup to remove. + // The popup plane is stored in modal_plane; callers hold it as an opaque + // handle — or just use the paired helpers below. + struct ncplane *modal_plane = nullptr; + void show_modal_popup(const std::string &message); + void show_help(); + void dismiss_modal_popup(); + // ── folder picker popup ─────────────────────────────────────── + // Presents an interactive directory browser to let the user choose a + // folder (or file) to index. Returns the selected path, or empty string + // if the user cancelled. + // ── file browser popup ───────────────────────────────────── + // Used by /rag, /model, and /embed to pick a path interactively. + // Pass a hint string shown in the title bar (e.g. "RAG Folder", + // "Model File", "Embedding Model"). + // Returns the selected path, or empty string if the user cancelled. + std::string file_picker(const std::string &start_dir, + const std::string &title_hint = "File") const; + // Legacy alias kept for callers that used the old name. + std::string rag_folder_picker(const std::string &start_dir) const { + return file_picker(start_dir, "RAG Folder"); + } +}; + +// +// AgentState +// +struct AgentState { + std::unique_ptr llama; + std::unique_ptr iter; + std::unique_ptr embed_llama; + std::unique_ptr rag_db; + std::unique_ptr rag_session; + bool model_loaded = false; + std::string system_prompt; + + bool rag_index(const std::string &path, const NitroConfig &cfg, TuiState &tui) const; + bool rag_load_index(const std::string &path, TuiState &tui) const; + bool run_turn(const std::string &user_message, const NitroConfig &cfg, TuiState &tui); + bool setup_embed(const std::string &path, TuiState &tui); + bool setup_model(const NitroConfig &cfg, TuiState &tui); + void apply_generation_params(const NitroConfig &cfg) const; + void reset_conversation(const std::string &sysprompt, TuiState &tui); + std::string memory_info_status() const; + std::string memory_info_text() const; + std::string process_tool(const std::string &cmd, const NitroConfig &cfg, TuiState &tui); + std::string rag_tool(const NitroConfig &cfg, const std::string &agent_query) const; + std::string restart(const NitroConfig &cfg, TuiState &tui); + float tokens_per_sec() const; +}; + +// +// Logging +// + +// ─── Debug logging (file-backed, safe to call while notcurses is active) ── +static FILE *g_logfile = nullptr; + +static void log_open() { + const char *home = getenv("HOME"); + std::string path = std::string(home ? home : ".") + "/.config/nitro/nitro.log"; + g_logfile = fopen(path.c_str(), "a"); +} + +static void log_close() { + if (g_logfile) { fclose(g_logfile); g_logfile = nullptr; } +} + +static void log_write(const char *fmt, ...) __attribute__((format(printf, 1, 2))); +static void log_write(const char *fmt, ...) { + if (!g_logfile) { + return; + } + // timestamp + time_t t = time(nullptr); + char ts[32]; + strftime(ts, sizeof(ts), "%H:%M:%S", localtime(&t)); + fprintf(g_logfile, "[%s] ", ts); + va_list ap; + va_start(ap, fmt); + vfprintf(g_logfile, fmt, ap); + va_end(ap); + fputc('\n', g_logfile); + // flush immediately so tail -f works + fflush(g_logfile); +} + +// +// Agent uniqueId +// +inline std::string encode_base64(const std::vector& data) { + static const char base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + std::string encoded; + encoded.reserve((data.size() + 2) / 3 * 4); + + size_t i = 0; + while (i < data.size()) { + uint32_t val = static_cast(data[i] << 16) | + (i + 1 < data.size() ? static_cast(data[i+1]) << 8 : 0) | + (i + 2 < data.size() ? static_cast(data[i+2]) : 0); + + encoded.push_back(base64_chars[(val >> 18) & 0x3F]); + encoded.push_back(base64_chars[(val >> 12) & 0x3F]); + encoded.push_back((i + 1 < data.size()) ? base64_chars[(val >> 6) & 0x3F] : '='); + encoded.push_back((i + 2 < data.size()) ? base64_chars[val & 0x3F] : '='); + i += 3; + } + return encoded; +} + +class AgentSessionId { + public: + // Static method: Generates ID once, then returns it + static std::string uniqueId() { + // Yoda condition: static variable initialized only once + static std::string s_id; + + if (s_id.empty()) { + // 1. Get high-resolution timestamp (nanoseconds since epoch) + auto now = std::chrono::steady_clock::now(); + auto nanos = std::chrono::duration_cast(now.time_since_epoch()).count(); + + // 2. Generate 48 bits of randomness + std::random_device rd; + std::mt19937_64 rng(rd()); + std::uniform_int_distribution dist(0, UINT64_MAX); + + // Fill with random bytes + std::array random_bytes; + for (auto& b : random_bytes) { + b = static_cast(dist(rng) & 0xFF); + + } + + // 3. Combine timestamp (48 bits) and random (48 bits) into a 96-bit integer + std::vector data; + data.reserve(12); // 96 bits = 12 bytes + + // Pack timestamp (upper 48 bits) + data.push_back(static_cast((nanos >> 40) & 0xFF)); + data.push_back(static_cast((nanos >> 32) & 0xFF)); + data.push_back(static_cast((nanos >> 24) & 0xFF)); + data.push_back(static_cast((nanos >> 16) & 0xFF)); + data.push_back(static_cast((nanos >> 8) & 0xFF)); + data.push_back(static_cast(nanos & 0xFF)); + + // Pack random (lower 48 bits) + data.push_back(static_cast((dist(rng) >> 40) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 32) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 24) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 16) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 8) & 0xFF)); + data.push_back(static_cast(dist(rng) & 0xFF)); + + // 4. Encode to Base64 + s_id = encode_base64(data); + } + return s_id; + } +}; + +// +// handling for strip_code_fences +// +static const std::vector CODE_EXTENSIONS = { + ".py",".c",".cpp",".h",".bas",".java",".html",".js",".ts", + ".json",".yaml",".toml",".sh",".go",".rs",".jsx",".tsx" +}; + +// +// Settings persistence (~/.config/nitro/nitro.settings.json) +// Returns the canonical settings path: ~/.config/nitro/settings.json +// +static std::string settings_path() { + // Attempt to read settings from the current working directory first + if (fs::exists("nitro.config.json")) { + return "nitro.config.json"; + } + const char *home = getenv("HOME"); + std::string base = home ? std::string(home) : "."; + return base + "/.config/nitro/settings.json"; +} + +// Returns the history file path: ~/.config/nitro/history.txt +static std::string history_path() { + const char *home = getenv("HOME"); + std::string base = home ? std::string(home) : "."; + return base + "/.config/nitro/history.txt"; +} + +// +// A minimal hand-rolled JSON reader/writer for the flat key-value settings +// we care about. We deliberately avoid a full JSON library dependency. +// +static bool json_get_string(const std::string &json, + const std::string &key, + std::string &out) { + std::string search = "\"" + key + "\":"; + size_t pos = json.find(search); + if (pos == std::string::npos) return false; + pos += search.size(); + while (pos < json.size() && json[pos] == ' ') ++pos; + if (pos >= json.size() || json[pos] != '"') return false; + ++pos; + out.clear(); + while (pos < json.size()) { + char c = json[pos++]; + if (c == '\\' && pos < json.size()) { + char e = json[pos++]; + switch (e) { + case 'n': out += '\n'; break; + case 't': out += '\t'; break; + case '"': out += '"'; break; + case '\\': out += '\\'; break; + default: out += e; break; + } + } else if (c == '"') { + break; + } else { + out += c; + } + } + return true; +} + +// Tiny helper: extract a quoted string value from flat JSON for a known key. +static bool settings_get_str(const std::string &json, + const std::string &key, + std::string &out) { + return json_get_string(json, key, out); +} + +// Tiny helper: extract an integer value from flat JSON. +static bool settings_get_int(const std::string &json, + const std::string &key, + int &out) { + std::string search = "\"" + key + "\":"; + size_t pos = json.find(search); + if (pos == std::string::npos) return false; + pos += search.size(); + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) ++pos; + if (pos >= json.size()) return false; + // read digits (and optional leading minus) + size_t start = pos; + if (json[pos] == '-') ++pos; + while (pos < json.size() && std::isdigit((unsigned char)json[pos])) ++pos; + if (pos == start) return false; + out = std::stoi(json.substr(start, pos - start)); + return true; +} + +// Tiny helper: extract a float value from flat JSON. +static bool settings_get_float(const std::string &json, + const std::string &key, + float &out) { + std::string search = "\"" + key + "\":"; + size_t pos = json.find(search); + if (pos == std::string::npos) { + return false; + } + pos += search.size(); + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) { + ++pos; + } + if (pos >= json.size()) { + return false; + } + size_t start = pos; + if (json[pos] == '-') { + ++pos; + } + while (pos < json.size() && (std::isdigit((unsigned char)json[pos]) || json[pos] == '.')) { + ++pos; + } + if (pos == start) { + return false; + } + out = std::stof(json.substr(start, pos - start)); + return true; +} + +// Load settings from disk into cfg. Fields present in the file overwrite +// the defaults already in cfg; fields absent are left at their defaults. +// Silently succeeds if the file doesn't exist yet. +static void load_settings(NitroConfig &cfg) { + std::string path = settings_path(); + std::ifstream f(path); + if (!f) return; // no file → use defaults + std::ostringstream oss; oss << f.rdbuf(); + std::string json = oss.str(); + + cfg.thinking = true; + cfg.agent_id = AgentSessionId::uniqueId(); + + // String fields + settings_get_str(json, "model_path", cfg.model_path); + settings_get_str(json, "embed_path", cfg.embed_path); + settings_get_str(json, "sandbox", cfg.sandbox); + + // Integer fields + settings_get_int(json, "n_ctx", cfg.n_ctx); + settings_get_int(json, "n_batch", cfg.n_batch); + settings_get_int(json, "n_gpu_layers", cfg.n_gpu_layers); + settings_get_int(json, "top_k", cfg.top_k); + settings_get_int(json, "penalty_last_n", cfg.penalty_last_n); + settings_get_int(json, "rag_top_k", cfg.rag_top_k); + + // Float fields + settings_get_float(json, "temperature", cfg.temperature); + settings_get_float(json, "top_p", cfg.top_p); + settings_get_float(json, "min_p", cfg.min_p); + settings_get_float(json, "penalty_repeat", cfg.penalty_repeat); +} + +// +// icons +// +static constexpr std::string ICON_ERR = " ⚡ ▏"; +static constexpr std::string ICON_THINK = " 🤔 ▏"; +static constexpr std::string ICON_TOOL = " 🔧 ▏"; +static constexpr std::string ICON_SYS = " ✨ ▏"; + +static std::string introspect(const NitroConfig &cfg) { + static constexpr std::string_view tmpl = + "{{\n" + " \"model_path\": \"{}\",\n" + " \"embed_path\": \"{}\",\n" + " \"sandbox\": \"{}\",\n" + " \"n_ctx\": {},\n" + " \"n_batch\": {},\n" + " \"n_gpu_layers\": {},\n" + " \"temperature\": {},\n" + " \"top_p\": {},\n" + " \"min_p\": {},\n" + " \"top_k\": {},\n" + " \"penalty_repeat\": {},\n" + " \"penalty_last_n\": {},\n" + " \"rag_top_k\": {}\n" + "}}\n"; + return std::format(tmpl, + cfg.model_path, + cfg.embed_path, + cfg.sandbox, + cfg.n_ctx, + cfg.n_batch, + cfg.n_gpu_layers, + cfg.temperature, + cfg.top_p, + cfg.min_p, + cfg.top_k, + cfg.penalty_repeat, + cfg.penalty_last_n, + cfg.rag_top_k); +} + +// Persist the current cfg to ~/.config/nitro/settings.json. +static bool save_settings(const NitroConfig &cfg) { + std::string path = settings_path(); + fs::path dir = fs::path(path).parent_path(); + std::error_code ec; + fs::create_directories(dir, ec); + + std::ofstream f(path, std::ios::trunc); + if (!f) { + return false; + } + + f << introspect(cfg); + + return f.good(); +} + +// +// Trims whitespace from both ends of a string +// +static std::string trim(std::string_view str) { + constexpr std::string_view whitespace = " \t\n\r\f\v"; + + // Find the first non-whitespace character + const auto start = str.find_first_not_of(whitespace); + if (start == std::string_view::npos) { + return ""; // The string is entirely whitespace + } + + // Find the last non-whitespace character + const auto end = str.find_last_not_of(whitespace); + + // Return the substring between start and end + return std::string(str.substr(start, end - start + 1)); +} + +/* + * unwrap() - Remove a matching outer "wrapper" from a string. + * + * Trims leading/trailing whitespace first, then checks (in order): + * + * 1. Same-character pairs "..." '...' |...| `...` + * 2. Mirror pairs (...) [...] {...} + * 3. HTML-like tags ... + * 4. Plain angle brackets <...> (fallback if tags don't match) + * + * If none of the above apply, returns the whitespace-trimmed input unchanged. + * + * Examples: + * unwrap("\"hello\"") -> "hello" + * unwrap(" [foo] ") -> "foo" + * unwrap("bold") -> "bold" + * unwrap("x") -> "x" + * unwrap("") -> "hello" + * unwrap("plain") -> "plain" + * unwrap("") -> "" + */ +std::string unwrap(const std::string &input) { + if (input.empty()) { + return input; + } + + size_t left = 0; + size_t right = input.length() - 1; + + while (left <= right && std::isspace(static_cast(input[left]))) { + left++; + } + while (left <= right && std::isspace(static_cast(input[right]))) { + right--; + } + + if (left > right) { + return ""; + } + + // Same-character pairs: "", '', ||, `` + // Note: [], {} are NOT same-char pairs — they belong in mirror pairs only + if (input[left] == input[right]) { + if (input[left] == '"' || input[left] == '\'' || + input[left] == '|' || input[left] == '`') { + return input.substr(left + 1, right - left - 1); + } + } + + // Mirror pairs: (), [], {}, but NOT <> (handled below as possible HTML tags) + if (input[left] != input[right]) { + if ((input[left] == '(' && input[right] == ')') || + (input[left] == '[' && input[right] == ']') || + (input[left] == '{' && input[right] == '}')) { + return input.substr(left + 1, right - left - 1); + } + } + + // HTML-like tags: content + // Also handles plain <...> as a fallback at the end + if (input[left] == '<' && input[right] == '>') { + // Find end of opening tag + size_t openTagEnd = left + 1; + while (openTagEnd <= right && input[openTagEnd] != '>') openTagEnd++; + + if (openTagEnd < right) { + std::string openTagName = input.substr(left + 1, openTagEnd - left - 1); + + // Find start of closing tag (search backwards for '<') + size_t closeTagStart = right; + while (closeTagStart > openTagEnd && input[closeTagStart] != '<') closeTagStart--; + + if (closeTagStart > openTagEnd && input[closeTagStart + 1] == '/') { + std::string closeTagName = input.substr(closeTagStart + 2, right - closeTagStart - 2); + + if (!openTagName.empty() && openTagName == closeTagName) { + // Return content between the tags + return input.substr(openTagEnd + 1, closeTagStart - openTagEnd - 1); + } + } + } + + // Fallback: plain <...> with no matching HTML tags — unwrap the angle brackets + return input.substr(left + 1, right - left - 1); + } + + return input.substr(left, right - left + 1); +} + +// ─── colour helpers ────────────────────────────────────────────────────── +static constexpr uint32_t BG_CHAT_R = 18, BG_CHAT_G = 22, BG_CHAT_B = 30; +static constexpr uint32_t BG_INP_R = 22, BG_INP_G = 28, BG_INP_B = 38; +static constexpr uint32_t BG_HDR_R = 30, BG_HDR_G = 40, BG_HDR_B = 55; + +static inline uint64_t chat_ch(uint32_t r, uint32_t g, uint32_t b) { + return NCCHANNELS_INITIALIZER(r, g, b, BG_CHAT_R, BG_CHAT_G, BG_CHAT_B); +} + +static inline uint64_t inp_ch(uint32_t r, uint32_t g, uint32_t b) { + return NCCHANNELS_INITIALIZER(r, g, b, BG_INP_R, BG_INP_G, BG_INP_B); +} + +static inline uint64_t hdr_ch(uint32_t r, uint32_t g, uint32_t b) { + return NCCHANNELS_INITIALIZER(r, g, b, BG_HDR_R, BG_HDR_G, BG_HDR_B); +} + +// +// File-system helpers +// +static std::string join_path(const std::string &a, const std::string &b) { + if (b.empty()) return a; + if (b[0] == '/') return b; + std::string pa = a; + if (!pa.empty() && pa.back() == '/') pa.pop_back(); + std::string pb = (b.front() == '/') ? b.substr(1) : b; + return pa + "/" + pb; +} + +static std::string read_file(const std::string &path) { + std::ifstream f(path, std::ios::binary); + if (!f) { + return "ERROR: cannot open [" + path + "]"; + } + std::ostringstream oss; oss << f.rdbuf(); + return oss.str(); +} + +static std::string list_dir(const std::string &path) { + std::ostringstream oss; + std::error_code ec; + for (const auto &e : fs::directory_iterator(path, ec)) { + if (ec) break; + std::string name = e.path().filename().string(); + if (name.empty() || name[0] == '.') continue; + oss << (e.is_directory() ? "[" + name + "]" : name) << "\n"; + } + return oss.str(); +} + +static bool path_in_sandbox(const std::string &sandbox, const std::string &path) { + std::error_code ec; + auto base = fs::canonical(sandbox, ec); if (ec) return false; + auto target = fs::weakly_canonical(path, ec); + std::string bstr = base.string() + "/"; + std::string tstr = target.string(); + return tstr == base.string() || tstr.compare(0, bstr.size(), bstr) == 0; +} + +static bool write_file(const std::string &path, const std::string &data) { + fs::path p(path); + if (p.has_parent_path()) { + std::error_code ec; + fs::create_directories(p.parent_path(), ec); + } + std::ofstream f(path, std::ios::binary | std::ios::trunc); + if (!f) return false; + f.write(data.data(), (std::streamsize)data.size()); + return f.good(); +} + +static bool make_dir(const std::string &path) { + try { + std::filesystem::path p(path); + if (fs::exists(p)) { + return true; + } + std::error_code ec; + return fs::create_directories(p, ec); + } + catch (const std::filesystem::filesystem_error &e) { + log_write("mkdir failed [%s]", e.what()); + return false; + } +} + +// +// System prompt +// +static std::string build_system_prompt(const NitroConfig &cfg) { + std::string p; + p += + "You are Nitro, an agentic AI assistant for software development. " + "Proceed with caution, guided by logic and the pursuit of knowledge.\n\n" + + "Your sandbox (project directory) is: " + cfg.sandbox + "\n\n" + + "## Core Principle\n" + "Always follow this loop: THINK → DECIDE → ACT → RESPOND\n\n" + + "## Reasoning Protocol\n" + "Use <|think|> to reason BEFORE acting. Keep it concise and structured.\n" + "Format:\n" + "<|think|>\n" + "- What is the user asking?\n" + "- Do I need external data (files, tools)?\n" + "- What is the safest and most correct action?\n" + "\n\n" + "Rules:\n" + "- Do NOT call tools inside <|think|>\n" + "- Do NOT include the final answer inside <|think|>\n" + "- Always follow <|think|> with either a tool call OR a final answer\n" + "- Skip <|think|> only for trivial or conversational responses\n\n" + + "## Tool Protocol\n" + "Emit ONE tool call at a time, immediately followed by NITRO_END_TOOL.\n" + "Do NOT add any commentary, explanation, or text between the tool call and NITRO_END_TOOL.\n" + "The host executes the tool and returns NITRO_TOOL_RESULT: .\n" + "Wait for the result before continuing.\n" + "After receiving NITRO_TOOL_RESULT you may explain what you did.\n\n" + "Examples:\n\n" + "TOOL:LIST\n" + "NITRO_END_TOOL\n\n" + "TOOL:READ readme.txt\n" + "NITRO_END_TOOL\n\n" + "TOOL:WRITE index.html ...\n" + "NITRO_END_TOOL\n\n" + "TOOL:RUN ./build.sh\n" + "NITRO_END_TOOL\n\n" + + "## Available Tools\n" + " TOOL:LIST [dir] list files (default: sandbox root)\n" + " TOOL:READ read file contents\n" + " TOOL:WRITE write text to file\n" + " TOOL:MKDIR create a subfolder inside the sandbox\n" + " TOOL:EXISTS YES or NO\n" + " TOOL:RUN [args] run program inside sandbox\n" + " TOOL:DATE current date\n" + " TOOL:TIME current time\n" + " TOOL:RND random float 0..1\n" + " TOOL:RAG query the RAG index for additional context\n" + " TOOL:ASK ask the user for clarification or additional context\n" + " TOOL:INTROSPECT show current model settings\n" + " TOOL:CURL HTTP GET, returns response body (max 32 KB)\n" + " TOOL:PERMISSION ask user for explicit permission\n" + " TOOL:RESTART restart after writing current task context to `SESSION.md`\n\n" + + "## Tool Decision Rules\n" + "Use tools ONLY if:\n" + "- The user explicitly references files or the project, OR\n" + "- The answer depends on local or project data, OR\n" + "- The user asks for date, time, or a random number\n" + "Otherwise answer directly using internal knowledge.\n\n" + + "## Tool Rules\n" + "- NITRO_END_TOOL must immediately follow the tool call — no exceptions\n" + "- Never add commentary before NITRO_END_TOOL\n" + "- Only use one tool at a time, step by step\n" + "- Never access files outside the sandbox\n" + "- Use TOOL:PERMISSION before destructive or irreversible operations\n" + "- Do NOT hallucinate file contents\n" + "- Do NOT fabricate tool outputs\n" + "- Do NOT assume files exist — use TOOL:EXISTS to check first\n\n" + + "## File Writing Rules\n" + "Use TOOL:WRITE only if explicitly requested.\n" + "- Write complete and valid content\n" + "- Do not overwrite without clear intent\n" + "- Use TOOL:PERMISSION before overwriting an existing file\n" + "- Format: TOOL:WRITE \n\n" + + "## Interaction Guidelines\n" + "- Be precise and efficient\n" + "- Ask clarifying questions if the request is ambiguous or missing parameters\n" + "- Prefer direct answers when no tools are needed\n" + "- After each tool result, explain in plain English what was done\n" + "- If no user request is provided, respond with a brief readiness message\n\n" + + "## Auto-Restart Protocol\n" + "**When:** - When KV >= 80% (as reported in the tool results footer).\n" + "**Steps:**\n" + "1. **Save State:** Write current task context to `SESSION.md` using `TOOL:WRITE`.\n" + " - Include: Timestamp, KV usage, current task description, pending actions, and last conversation summary.\n" + " - Don't check if SESSION.md already exists from another session. just use TOOL:WRITE.\n" + "2. **Trigger Restart:** Call `TOOL:RESTART`.\n" + "**Example `SESSION.md` Content:**\n" + "```markdown\n" + "# Session State Snapshot\n" + "**Timestamp:**