Skip to content

feat(vision): add vision tool using MiniCPM-V 4.6 (1.3B) via llama-mtmd-cli #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jkyberneees merged 2 commits into from
Jun 7, 2026
Merged

feat(vision): add vision tool using MiniCPM-V 4.6 (1.3B) via llama-mtmd-cli #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b106eed
feat(vision): add vision tool using MiniCPM-V 4.6 (1.3B) via llama-mt…
jkyberneees Jun 7, 2026
be67d47
fix(vision): pin MiniCPM-V HuggingFace downloads to a specific revisi…
jkyberneees Jun 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ odek is not a framework. It's a **runtime** — the smallest possible surface ar
Every session can run in an isolated Docker container: no network, no host mounts beyond the working directory, zero capabilities, destroyed on exit. `odek serve` enables the sandbox **by default**; `odek run` keeps it opt-in but warns when running unsandboxed. `--ctx` files are auto-injected into the container at `/workspace/`. Full security model in [docs/SANDBOXING.md](docs/SANDBOXING.md).

### 🛡️ Prompt-Injection-Aware
External content the agent ingests (`browser`, `read_file`, `shell`, `search_files`, `multi_grep`, `transcribe`, `session_search`, MCP tools) is wrapped in per-call nonce'd `<untrusted_content>` boundaries so the model can distinguish data from instructions. Redirect hops are re-classified (`browser`/`http_batch`), MCP tool descriptions are scanned for injection at registration, and the MCP error channel is wrapped too. The danger classifier resists 8 known shell-evasion tricks (`$()`, backticks, `$IFS`, `command`/`exec`, `\rm`, basenamed absolute paths). Approvers engage friction mode after 3 same-class approvals in 60 s. Memory episodes from tainted sessions are stored but never auto-replayed. Skill auto-save tracks provenance and pins untrusted suggestions for explicit `odek skill promote`. `odek audit <session-id>` surfaces every ingest + per-turn divergence heuristic. Full threat model in [docs/SECURITY.md](docs/SECURITY.md).
External content the agent ingests (`browser`, `read_file`, `shell`, `search_files`, `multi_grep`, `transcribe`, `vision`, `session_search`, MCP tools) is wrapped in per-call nonce'd `<untrusted_content>` boundaries so the model can distinguish data from instructions. Redirect hops are re-classified (`browser`/`http_batch`), MCP tool descriptions are scanned for injection at registration, and the MCP error channel is wrapped too. The danger classifier resists 8 known shell-evasion tricks (`$()`, backticks, `$IFS`, `command`/`exec`, `\rm`, basenamed absolute paths). Approvers engage friction mode after 3 same-class approvals in 60 s. Memory episodes from tainted sessions are stored but never auto-replayed. Skill auto-save tracks provenance and pins untrusted suggestions for explicit `odek skill promote`. `odek audit <session-id>` surfaces every ingest + per-turn divergence heuristic. Full threat model in [docs/SECURITY.md](docs/SECURITY.md).

### 🧩 Sub-Agent Delegation
Parallel OS-process sub-agents via `delegate_tasks`. True isolation — each sub-agent is a fresh `odek subagent` process with its own config, tools, and termination timeout. Up to 8 concurrent workers. [docs/SUBAGENTS.md](docs/SUBAGENTS.md)
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/injection_hardening_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ func TestBuiltinTools_SessionSearchWrappedAsUntrusted(t *testing.T) {
store, cleanup := seedSessionStore(t)
defer cleanup()

tools := builtinTools(danger.DangerousConfig{}, nil, nil, 4, "", config.TranscriptionConfig{}, store)
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 4, "", config.TranscriptionConfig{}, config.VisionConfig{}, store)

var ss odek.Tool
for _, tool := range tools {
Expand Down
7 changes: 4 additions & 3 deletions cmd/odek/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -779,7 +779,7 @@ func run(args []string) error {

// Sandbox setup
var sandboxCleanup func() error
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, nil)
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, resolved.Vision, nil)

// MCP server tools
var mcpCleanup func()
Expand Down Expand Up @@ -1054,7 +1054,7 @@ func setupSandbox(tools []odek.Tool, cfg sandboxConfig) (containerName string, c
return containerName, cleanup, nil
}

func builtinTools(dc danger.DangerousConfig, sm *skills.SkillManager, approver danger.Approver, maxConcurrency int, apiKey string, tc config.TranscriptionConfig, store *session.Store) []odek.Tool {
func builtinTools(dc danger.DangerousConfig, sm *skills.SkillManager, approver danger.Approver, maxConcurrency int, apiKey string, tc config.TranscriptionConfig, vc config.VisionConfig, store *session.Store) []odek.Tool {
tools := []odek.Tool{
&shellTool{
dangerousConfig: dc,
Expand Down Expand Up @@ -1089,6 +1089,7 @@ func builtinTools(dc danger.DangerousConfig, sm *skills.SkillManager, approver d
&trTool{dangerousConfig: dc},
&wordCountTool{dangerousConfig: dc},
newTranscribeTool(dc, tc),
newVisionTool(dc, vc),
// session_search returns content from arbitrary past sessions —
// including sessions that ingested untrusted content. That path
// otherwise bypasses the memory taint gate and the audit log, so
Expand Down Expand Up @@ -1598,7 +1599,7 @@ func continueCmd(args []string) error {
"./.odek/skills",
)
}
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, store)
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, resolved.Vision, store)
var sandboxCleanup func() error

// MCP server tools
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func TestRun_NoAPIKey(t *testing.T) {
}

func TestBuiltinTools(t *testing.T) {
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, nil)
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, config.VisionConfig{}, nil)
if len(tools) == 0 {
t.Fatal("builtinTools() returned empty slice")
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/mcp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ Flags:
}

// Build tools
toolSet := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, nil)
toolSet := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, config.VisionConfig{}, nil)

// MCP server tools — connect and discover before sandbox
var mcpCleanup func()
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/repl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func replCmd(args []string) error {
"./.odek/skills",
)
}
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, nil)
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, config.VisionConfig{}, nil)
var sandboxCleanup func() error

// MCP server tools
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/schedule.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,7 @@ func runTaskHeadless(ctx context.Context, resolved config.ResolvedConfig, system
resolved.Dangerous.NonInteractive = &deny
}

tools := builtinTools(resolved.Dangerous, nil, nil, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, nil)
tools := builtinTools(resolved.Dangerous, nil, nil, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, resolved.Vision, nil)
tools = append(tools, mcpTools...)

// Capture cumulative token usage from the final iteration so the Runner
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/serve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ func newServeAgent(resolved config.ResolvedConfig, system string, sendFn func(v
approver := newWSApprover(sendFn)
resolved.Dangerous.Approver = approver

tools := builtinTools(resolved.Dangerous, sm, approver, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, nil)
tools := builtinTools(resolved.Dangerous, sm, approver, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, config.VisionConfig{}, nil)

// Find the delegateTasksTool to wire up sub-agent log streaming
var subagentTool *delegateTasksTool
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/subagent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ func subagentCmd(args []string) error {
"./.odek/skills",
)
}
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, nil)
tools := builtinTools(resolved.Dangerous, sm, nil, resolved.MaxConcurrency, resolved.APIKey, config.TranscriptionConfig{}, config.VisionConfig{}, nil)
var sandboxCleanup func() error

// MCP server tools
Expand Down
6 changes: 3 additions & 3 deletions cmd/odek/subagent_contract_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ func TestSubagent_ExitCodeThree(t *testing.T) {
// ── 4. delegate_tasks Tool Schema ───────────────────────────────────

func TestDelegateTasksTool_Exists(t *testing.T) {
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, nil)
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, config.VisionConfig{}, nil)
if len(tools) == 0 {
t.Fatal("builtinTools() returned empty slice")
}
Expand All @@ -338,7 +338,7 @@ func TestDelegateTasksTool_Exists(t *testing.T) {
}

func TestDelegateTasksTool_HasSchema(t *testing.T) {
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, nil)
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, config.VisionConfig{}, nil)

var tool odek.Tool
for _, t2 := range tools {
Expand Down Expand Up @@ -432,7 +432,7 @@ func TestDelegateTasksTool_HasSchema(t *testing.T) {
}

func TestDelegateTasksTool_Description(t *testing.T) {
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, nil)
tools := builtinTools(danger.DangerousConfig{}, nil, nil, 3, "", config.TranscriptionConfig{}, config.VisionConfig{}, nil)

var tool odek.Tool
for _, t2 := range tools {
Expand Down
2 changes: 1 addition & 1 deletion cmd/odek/telegram.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1078,7 +1078,7 @@ func handleChatMessage(
}

// Build the agent with Telegram approver.
tools := builtinTools(resolved.Dangerous, nil, approver, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, sessionManager.Store)
tools := builtinTools(resolved.Dangerous, nil, approver, resolved.MaxConcurrency, resolved.APIKey, resolved.Transcription, resolved.Vision, sessionManager.Store)

modelLabel := odek.ProfileLabel(resolved.Model)
if modelLabel == "" {
Expand Down
Loading
Loading