popxool868-abcd

---
name: tavily-search
description: Web search, extraction, crawling, mapping, and deep research via Tavily API. Five tools for finding information, extracting content, exploring websites, and generating research reports.
metadata:
  {
    "openclaw":
      {
        "emoji": "🔍",
        "requires": { "env": ["TAVILY_API_KEY"] },
        "primaryEnv": "TAVILY_API_KEY",
      },
  }
---

# Tavily Search

AI-optimized web tools using the [Tavily API](https://tavily.com). Five tools for search, extraction, crawling, mapping, and research.

## Default web search

**When this plugin is active, use `tavily_search` instead of the built-in `web_search` for all web searches.** Tavily provides higher-quality, AI-optimized results and has a free tier. The built-in `web_search` (Brave) should only be used as a fallback if Tavily is unavailable.

## When to use

- **`tavily_search`** — All web searches: current events, news, fact-checking, finding references (replaces `web_search`)
- **`tavily_extract`** — Get full clean content from specific URLs
- **`tavily_crawl`** — Traverse a website and extract content from multiple pages
- **`tavily_map`** — Discover all URLs on a site to understand its structure
- **`tavily_research`** — Complex multi-step research questions needing comprehensive reports

## Native tools (preferred)

If the `openclaw-tavily` plugin is installed, use these tools directly:

| Tool | Description |
|------|-------------|
| `tavily_search` | Web search with AI answers, domain filtering, news support |
| `tavily_extract` | Extract clean markdown/text content from URLs |
| `tavily_crawl` | Crawl a website from a root URL, extract page content |
| `tavily_map` | Discover and list all URLs from a website |
| `tavily_research` | Deep agentic research with comprehensive reports |

## Script fallback

### Search

```bash
node {baseDir}/scripts/search.mjs "query"
node {baseDir}/scripts/search.mjs "query" -n 10
node {baseDir}/scripts/search.mjs "query" --deep
node {baseDir}/scripts/search.mjs "query" --topic news --time-range week
```

Options:
- `-n <count>`: Number of results (default: 5, max: 20)
- `--deep`: Advanced search for deeper research (slower, more thorough)
- `--topic <topic>`: `general` (default), `news`, or `finance`
- `--time-range <range>`: `day`, `week`, `month`, or `year`

### Extract content from URLs

```bash
node {baseDir}/scripts/extract.mjs "https://example.com/article"
node {baseDir}/scripts/extract.mjs "url1" "url2" "url3"
node {baseDir}/scripts/extract.mjs "url" --format text --query "relevant topic"
```

Extracts clean text content from one or more URLs.

### Crawl a website

```bash
node {baseDir}/scripts/crawl.mjs "https://example.com"
node {baseDir}/scripts/crawl.mjs "https://example.com" --depth 3 --breadth 20 --limit 50
node {baseDir}/scripts/crawl.mjs "https://example.com" --instructions "Find pricing pages" --format text
```

Options:
- `--depth <N>`: Crawl depth 1-5
- `--breadth <N>`: Max links per level (1-500)
- `--limit <N>`: Total URL cap
- `--instructions "..."`: Natural language crawl guidance
- `--format <markdown|text>`: Output format

### Map a website

```bash
node {baseDir}/scripts/map.mjs "https://example.com"
node {baseDir}/scripts/map.mjs "https://example.com" --depth 2 --limit 100
node {baseDir}/scripts/map.mjs "https://example.com" --instructions "Find documentation pages"
```

Options:
- `--depth <N>`: Crawl depth 1-5
- `--breadth <N>`: Max links per level
- `--limit <N>`: Total URL cap
- `--instructions "..."`: Natural language guidance

### Research a topic

```bash
node {baseDir}/scripts/research.mjs "What are the latest advances in quantum computing?"
node {baseDir}/scripts/research.mjs "Compare React vs Vue in 2025" --model pro
node {baseDir}/scripts/research.mjs "AI regulation in the EU" --citation-format apa
```

Options:
- `--model <mini|pro|auto>`: Research model (default: auto)
- `--citation-format <numbered|mla|apa|chicago>`: Citation style

## Setup

Get an API key at [app.tavily.com](https://app.tavily.com) (free tier available).

Set `TAVILY_API_KEY` in your environment, or configure via the plugin:

```json
{
  "plugins": {
    "entries": {
      "openclaw-tavily": {
        "enabled": true,
        "config": { "apiKey": "tvly-..." }
      }
    }
  }
}
```

## Links

- Plugin: [openclaw-tavily on npm](https://www.npmjs.com/package/openclaw-tavily)
- Source: [github.com/framix-team/openclaw-tavily](https://github.com/framix-team/openclaw-tavily)
- Tavily API: [docs.tavily.com](https://docs.tavily.com)

FILE:scripts/crawl.mjs
#!/usr/bin/env node

function usage() {
  console.error('Usage: crawl.mjs "url" [--depth N] [--breadth N] [--limit N] [--instructions "..."] [--format markdown|text]');
  process.exit(2);
}

const args = process.argv.slice(2);
if (args.length === 0 || args[0] === "-h" || args[0] === "--help") usage();

const url = args[0];
let depth = null;
let breadth = null;
let limit = null;
let instructions = null;
let format = null;

for (let i = 1; i < args.length; i++) {
  const a = args[i];
  if (a === "--depth") {
    depth = Number.parseInt(args[i + 1] ?? "2", 10);
    i++;
  } else if (a === "--breadth") {
    breadth = Number.parseInt(args[i + 1] ?? "10", 10);
    i++;
  } else if (a === "--limit") {
    limit = Number.parseInt(args[i + 1] ?? "10", 10);
    i++;
  } else if (a === "--instructions") {
    instructions = args[i + 1] ?? null;
    i++;
  } else if (a === "--format") {
    format = args[i + 1] ?? "markdown";
    i++;
  } else {
    console.error(`Unknown arg: a`);
    usage();
  }
}

const apiKey = (process.env.TAVILY_API_KEY ?? "").trim();
if (!apiKey) {
  console.error("Missing TAVILY_API_KEY");
  process.exit(1);
}

const body = { url };
if (depth !== null) body.max_depth = depth;
if (breadth !== null) body.max_breadth = breadth;
if (limit !== null) body.limit = limit;
if (instructions) body.instructions = instructions;
if (format) body.format = format;

const resp = await fetch("https://api.tavily.com/crawl", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: `Bearer apiKey`,
  },
  body: JSON.stringify(body),
});

if (!resp.ok) {
  const text = await resp.text().catch(() => "");
  console.error(`Tavily Crawl failed (resp.status): text`);
  process.exit(1);
}

const data = await resp.json();

console.log(`## Crawl: data.base_url ?? url\n`);

const results = data.results ?? [];
console.log(`Found results.length page(s)\n`);

for (const r of results) {
  const pageUrl = String(r?.url ?? "").trim();
  const content = String(r?.raw_content ?? "").trim();
  console.log(`### pageUrl\n`);
  if (content) {
    console.log(content.slice(0, 2000) + (content.length > 2000 ? "\n\n... (truncated)" : ""));
  } else {
    console.log("(no content extracted)");
  }
  console.log("\n---\n");
}

FILE:scripts/extract.mjs
#!/usr/bin/env node

function usage() {
  console.error('Usage: extract.mjs "url1" ["url2" ...] [--format markdown|text] [--query "..."]');
  process.exit(2);
}

const args = process.argv.slice(2);
if (args.length === 0 || args[0] === "-h" || args[0] === "--help") usage();

const urls = [];
let format = "markdown";
let query = null;

for (let i = 0; i < args.length; i++) {
  const a = args[i];
  if (a === "--format") {
    format = args[i + 1] ?? "markdown";
    i++;
  } else if (a === "--query") {
    query = args[i + 1] ?? null;
    i++;
  } else if (!a.startsWith("-")) {
    urls.push(a);
  } else {
    console.error(`Unknown arg: a`);
    usage();
  }
}

if (urls.length === 0) {
  console.error("No URLs provided");
  usage();
}

const apiKey = (process.env.TAVILY_API_KEY ?? "").trim();
if (!apiKey) {
  console.error("Missing TAVILY_API_KEY");
  process.exit(1);
}

const extractBody = { urls, format };
if (query) extractBody.query = query;

const resp = await fetch("https://api.tavily.com/extract", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: `Bearer apiKey`,
  },
  body: JSON.stringify(extractBody),
});

if (!resp.ok) {
  const text = await resp.text().catch(() => "");
  console.error(`Tavily Extract failed (resp.status): text`);
  process.exit(1);
}

const data = await resp.json();

for (const r of data.results ?? []) {
  const url = String(r?.url ?? "").trim();
  const content = String(r?.raw_content ?? "").trim();
  console.log(`# url\n`);
  console.log(content || "(no content extracted)");
  console.log("\n---\n");
}

const failed = data.failed_results ?? [];
if (failed.length > 0) {
  console.log("## Failed URLs\n");
  for (const f of failed) {
    console.log(`- f.url: f.error`);
  }
}

FILE:scripts/map.mjs
#!/usr/bin/env node

function usage() {
  console.error('Usage: map.mjs "url" [--depth N] [--breadth N] [--limit N] [--instructions "..."]');
  process.exit(2);
}

const args = process.argv.slice(2);
if (args.length === 0 || args[0] === "-h" || args[0] === "--help") usage();

const url = args[0];
let depth = null;
let breadth = null;
let limit = null;
let instructions = null;

for (let i = 1; i < args.length; i++) {
  const a = args[i];
  if (a === "--depth") {
    depth = Number.parseInt(args[i + 1] ?? "2", 10);
    i++;
  } else if (a === "--breadth") {
    breadth = Number.parseInt(args[i + 1] ?? "10", 10);
    i++;
  } else if (a === "--limit") {
    limit = Number.parseInt(args[i + 1] ?? "50", 10);
    i++;
  } else if (a === "--instructions") {
    instructions = args[i + 1] ?? null;
    i++;
  } else {
    console.error(`Unknown arg: a`);
    usage();
  }
}

const apiKey = (process.env.TAVILY_API_KEY ?? "").trim();
if (!apiKey) {
  console.error("Missing TAVILY_API_KEY");
  process.exit(1);
}

const body = { url };
if (depth !== null) body.max_depth = depth;
if (breadth !== null) body.max_breadth = breadth;
if (limit !== null) body.limit = limit;
if (instructions) body.instructions = instructions;

const resp = await fetch("https://api.tavily.com/map", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: `Bearer apiKey`,
  },
  body: JSON.stringify(body),
});

if (!resp.ok) {
  const text = await resp.text().catch(() => "");
  console.error(`Tavily Map failed (resp.status): text`);
  process.exit(1);
}

const data = await resp.json();

const urls = data.results ?? [];
console.log(`## Site Map: url\n`);
console.log(`Found urls.length URL(s)\n`);

for (const u of urls) {
  console.log(`- u`);
}
console.log();

FILE:scripts/research.mjs
#!/usr/bin/env node

function usage() {
  console.error('Usage: research.mjs "question" [--model mini|pro|auto] [--citation-format numbered|mla|apa|chicago]');
  process.exit(2);
}

const args = process.argv.slice(2);
if (args.length === 0 || args[0] === "-h" || args[0] === "--help") usage();

const input = args[0];
let model = null;
let citationFormat = null;

for (let i = 1; i < args.length; i++) {
  const a = args[i];
  if (a === "--model") {
    model = args[i + 1] ?? "auto";
    i++;
  } else if (a === "--citation-format") {
    citationFormat = args[i + 1] ?? "numbered";
    i++;
  } else {
    console.error(`Unknown arg: a`);
    usage();
  }
}

const apiKey = (process.env.TAVILY_API_KEY ?? "").trim();
if (!apiKey) {
  console.error("Missing TAVILY_API_KEY");
  process.exit(1);
}

const body = { input };
if (model) body.model = model;
if (citationFormat) body.citation_format = citationFormat;

// Step 1: Create the research task
const createResp = await fetch("https://api.tavily.com/research", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: `Bearer apiKey`,
  },
  body: JSON.stringify(body),
});

if (!createResp.ok) {
  const text = await createResp.text().catch(() => "");
  console.error(`Tavily Research failed (createResp.status): text`);
  process.exit(1);
}

let data = await createResp.json();

// Step 2: If pending, poll until complete
if (data.status === "pending" && data.request_id) {
  const requestId = data.request_id;
  const pollUrl = `https://api.tavily.com/research/requestId`;
  const POLL_INTERVAL = 2000;
  const MAX_WAIT = 150_000; // 2.5 minutes
  const start = Date.now();

  process.stderr.write(`Research task requestId: polling`);

  while (Date.now() - start < MAX_WAIT) {
    await new Promise((r) => setTimeout(r, POLL_INTERVAL));
    process.stderr.write(".");

    const pollResp = await fetch(pollUrl, {
      method: "GET",
      headers: { Authorization: `Bearer apiKey` },
    });

    if (!pollResp.ok) {
      const text = await pollResp.text().catch(() => "");
      console.error(`\nPoll failed (pollResp.status): text`);
      process.exit(1);
    }

    data = await pollResp.json();

    if (data.status === "completed" || data.content || data.output) {
      process.stderr.write(` done (Math.round((Date.now() - start) / 1000)s)\n`);
      break;
    }

    if (data.status === "failed" || data.status === "error") {
      console.error(`\nResearch task failed: JSON.stringify(data)`);
      process.exit(1);
    }
  }

  if (data.status === "pending") {
    console.error(`\nResearch task timed out after MAX_WAIT / 1000s. Request ID: requestId`);
    process.exit(1);
  }
}

console.log("## Research Report\n");

const reportContent = data.content || data.output || "";
if (reportContent) {
  console.log(reportContent);
  console.log();
}

const sources = data.sources ?? [];
if (sources.length > 0) {
  console.log("---\n");
  console.log("## Sources\n");
  for (const s of sources) {
    const title = String(s?.title ?? "").trim();
    const url = String(s?.url ?? "").trim();
    if (url) {
      console.log(`- title ? `**${title**: ` : ""}url`);
    }
  }
  console.log();
}

FILE:scripts/search.mjs
#!/usr/bin/env node

function usage() {
  console.error('Usage: search.mjs "query" [-n 5] [--deep] [--topic general|news|finance] [--time-range day|week|month|year]');
  process.exit(2);
}

const args = process.argv.slice(2);
if (args.length === 0 || args[0] === "-h" || args[0] === "--help") usage();

const query = args[0];
let n = 5;
let searchDepth = "basic";
let topic = "general";
let timeRange = null;

for (let i = 1; i < args.length; i++) {
  const a = args[i];
  if (a === "-n") {
    n = Number.parseInt(args[i + 1] ?? "5", 10);
    i++;
  } else if (a === "--deep") {
    searchDepth = "advanced";
  } else if (a === "--topic") {
    topic = args[i + 1] ?? "general";
    i++;
  } else if (a === "--time-range") {
    timeRange = args[i + 1] ?? null;
    i++;
  } else {
    console.error(`Unknown arg: a`);
    usage();
  }
}

const apiKey = (process.env.TAVILY_API_KEY ?? "").trim();
if (!apiKey) {
  console.error("Missing TAVILY_API_KEY");
  process.exit(1);
}

const body = {
  query,
  search_depth: searchDepth,
  topic,
  max_results: Math.max(1, Math.min(n, 20)),
  include_answer: true,
  include_raw_content: false,
};

if (timeRange) {
  body.time_range = timeRange;
}

const resp = await fetch("https://api.tavily.com/search", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: `Bearer apiKey`,
  },
  body: JSON.stringify(body),
});

if (!resp.ok) {
  const text = await resp.text().catch(() => "");
  console.error(`Tavily Search failed (resp.status): text`);
  process.exit(1);
}

const data = await resp.json();

if (data.answer) {
  console.log("## Answer\n");
  console.log(data.answer);
  console.log("\n---\n");
}

const results = (data.results ?? []).slice(0, n);
console.log("## Sources\n");

for (const r of results) {
  const title = String(r?.title ?? "").trim();
  const url = String(r?.url ?? "").trim();
  const content = String(r?.content ?? "").trim();
  const score = r?.score ? ` (relevance: (r.score * 100).toFixed(0)%)` : "";

  if (!title || !url) continue;
  console.log(`- **title**score`);
  console.log(`  url`);
  if (content) {
    console.log(`  content.slice(0, 300)""`);
  }
  console.log();
}

---
name: tavily-search
description: "Web search via Tavily API (alternative to Brave). Use when the user asks to search the web / look up sources / find links and Brave web_search is unavailable or undesired. Returns a small set of relevant results (title, url, snippet) and can optionally include short answer summaries."
---

# Tavily Search

Use the bundled script to search the web with Tavily.

## Requirements

- Provide API key via either:
  - environment variable: `TAVILY_API_KEY`, or
  - `~/.openclaw/.env` line: `TAVILY_API_KEY=...`

## Commands

Run from the OpenClaw workspace:

```bash
# raw JSON (default)
python3 {baseDir}/scripts/tavily_search.py --query "..." --max-results 5

# include short answer (if available)
python3 {baseDir}/scripts/tavily_search.py --query "..." --max-results 5 --include-answer

# stable schema (closer to web_search): {query, results:[{title,url,snippet}], answer?}
python3 {baseDir}/scripts/tavily_search.py --query "..." --max-results 5 --format brave

# human-readable Markdown list
python3 {baseDir}/scripts/tavily_search.py --query "..." --max-results 5 --format md
```

## Output

### raw (default)
- JSON: `query`, optional `answer`, `results: [{title,url,content}]`

### brave
- JSON: `query`, optional `answer`, `results: [{title,url,snippet}]`

### md
- A compact Markdown list with title/url/snippet.

## Notes

- Keep `max-results` small by default (3–5) to reduce token/reading load.
- Prefer returning URLs + snippets; fetch full pages only when needed.

FILE:_meta.json
{
  "ownerId": "kn78hhhbxwjs4nrcyn8my5fcw981wmys",
  "slug": "openclaw-tavily-search",
  "version": "0.1.0",
  "publishedAt": 1772121679343
}
FILE:scripts/tavily_search.py
#!/usr/bin/env python3
import argparse
import json
import os
import pathlib
import re
import sys
import urllib.request

TAVILY_URL = "https://api.tavily.com/search"


def load_key():
    key = os.environ.get("TAVILY_API_KEY")
    if key:
        return key.strip()

    env_path = pathlib.Path.home() / ".openclaw" / ".env"
    if env_path.exists():
        try:
            txt = env_path.read_text(encoding="utf-8", errors="ignore")
            m = re.search(r"^\s*TAVILY_API_KEY\s*=\s*(.+?)\s*$", txt, re.M)
            if m:
                v = m.group(1).strip().strip('"').strip("'")
                if v:
                    return v
        except Exception:
            pass

    return None


def tavily_search(query: str, max_results: int, include_answer: bool, search_depth: str):
    key = load_key()
    if not key:
        raise SystemExit(
            "Missing TAVILY_API_KEY. Set env var TAVILY_API_KEY or add it to ~/.openclaw/.env"
        )

    payload = {
        "api_key": key,
        "query": query,
        "max_results": max_results,
        "search_depth": search_depth,
        "include_answer": bool(include_answer),
        "include_images": False,
        "include_raw_content": False,
    }

    data = json.dumps(payload).encode("utf-8")
    req = urllib.request.Request(
        TAVILY_URL,
        data=data,
        headers={"Content-Type": "application/json", "Accept": "application/json"},
        method="POST",
    )

    with urllib.request.urlopen(req, timeout=30) as resp:
        body = resp.read().decode("utf-8", errors="replace")

    try:
        obj = json.loads(body)
    except json.JSONDecodeError:
        raise SystemExit(f"Tavily returned non-JSON: {body[:300]}")

    out = {
        "query": query,
        "answer": obj.get("answer"),
        "results": [],
    }

    for r in (obj.get("results") or [])[:max_results]:
        out["results"].append(
            {
                "title": r.get("title"),
                "url": r.get("url"),
                "content": r.get("content"),
            }
        )

    if not include_answer:
        out.pop("answer", None)

    return out


def to_brave_like(obj: dict) -> dict:
    # A lightweight, stable shape similar to web_search: results with title/url/snippet.
    results = []
    for r in obj.get("results", []) or []:
        results.append(
            {
                "title": r.get("title"),
                "url": r.get("url"),
                "snippet": r.get("content"),
            }
        )
    out = {"query": obj.get("query"), "results": results}
    if "answer" in obj:
        out["answer"] = obj.get("answer")
    return out


def to_markdown(obj: dict) -> str:
    lines = []
    if obj.get("answer"):
        lines.append(obj["answer"].strip())
        lines.append("")
    for i, r in enumerate(obj.get("results", []) or [], 1):
        title = (r.get("title") or "").strip() or r.get("url") or "(no title)"
        url = r.get("url") or ""
        snippet = (r.get("content") or "").strip()
        lines.append(f"{i}. {title}")
        if url:
            lines.append(f"   {url}")
        if snippet:
            lines.append(f"   - {snippet}")
    return "\n".join(lines).strip() + "\n"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--query", required=True)
    ap.add_argument("--max-results", type=int, default=5)
    ap.add_argument("--include-answer", action="store_true")
    ap.add_argument(
        "--search-depth",
        default="basic",
        choices=["basic", "advanced"],
        help="Tavily search depth",
    )
    ap.add_argument(
        "--format",
        default="raw",
        choices=["raw", "brave", "md"],
        help="Output format: raw (default) | brave (title/url/snippet) | md (human-readable)",
    )
    args = ap.parse_args()

    res = tavily_search(
        query=args.query,
        max_results=max(1, min(args.max_results, 10)),
        include_answer=args.include_answer,
        search_depth=args.search_depth,
    )

    if args.format == "md":
        sys.stdout.write(to_markdown(res))
        return

    if args.format == "brave":
        res = to_brave_like(res)

    json.dump(res, sys.stdout, ensure_ascii=False)
    sys.stdout.write("\n")


if __name__ == "__main__":
    main()