From 81a3fd318894fb03246d5f970c23dc1754fab4d8 Mon Sep 17 00:00:00 2001
From: anon <spywarezzz@proton.me>
Date: Mon, 8 Jul 2024 21:56:30 -0700
Subject: [PATCH] default to all true, open in new buffers for long queries

---
 README.md       | 19 +++++++++++++------
 lua/crawler.lua | 31 +++++++++++++++++++++----------
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index d803639..4af450c 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,14 @@
 # crawler.nvim
 
-A Neovim plugin for crawling web pages, rendering them to Markdown or JSON, and inserting the content directly into your buffer. It also supports search functionality.
+A Neovim plugin for crawling web pages, rendering them to Markdown or JSON, and inserting the content into new buffers. It also supports asynchronous search functionality.
 
 ## Features
 
 - Process single URLs, multiple URLs, or search queries
 - Render web pages to Markdown or JSON
-- Insert processed content directly into your Neovim buffer
+- Insert processed content into new Neovim buffers
 - Supports visual selection or manual input
+- Asynchronous search functionality
 - Configurable options for rendering and search functionality
 
 ## Installation
@@ -28,12 +29,12 @@ Using [lazy.nvim](https://github.com/folke/lazy.nvim):
 
 ## Configuration
 
-You can configure the plugin by passing options to the `setup` function:
+You can configure the plugin by passing options to the `setup` function. The default configuration is:
 
 ```lua
 require("crawler").setup({
   render_markdown = true,  -- Enable markdown rendering (default: true)
-  render_json = false,     -- Enable JSON rendering (default: false)
+  render_json = true,      -- Enable JSON rendering (default: true)
   search_engine = true,    -- Enable search engine functionality (default: true)
 })
 ```
@@ -44,7 +45,7 @@ The plugin provides three main commands:
 
 - `:CrawlMarkdown`: Crawl a URL and render it to Markdown
 - `:CrawlJson`: Crawl a URL and render it to JSON
-- `:CrawlSearch`: Perform a search query
+- `:CrawlSearch`: Perform an asynchronous search query
 
 These commands can be used in normal mode (prompting for input) or visual mode (using the selected text as input).
 
@@ -78,11 +79,12 @@ You can override these mappings in your Neovim configuration if desired.
    > Enter URL or search query: https://example.com, https://another-example.com
    ```
 
-3. Perform a search:
+3. Perform an asynchronous search:
    ```
    :CrawlSearch
    > Enter search query: neovim lua plugins
    ```
+   The search will run in the background, and results will be displayed in a new buffer when ready.
 
 4. Using visual mode:
    - Select a URL or text in visual mode
@@ -90,6 +92,11 @@ You can override these mappings in your Neovim configuration if desired.
    - Press `<leader>lj` to crawl and render to JSON
    - Press `<leader>ls` to search using the selected text
 
+## Behavior
+
+- All crawled content and search results are opened in new buffers.
+- Search queries are processed asynchronously, allowing you to continue working while waiting for results.
+
 ## Integration with Other Tools
 
 crawler.nvim is particularly useful when used in conjunction with other plugins and tools that leverage Neovim buffers for various purposes:
diff --git a/lua/crawler.lua b/lua/crawler.lua
index c1080c1..bc117f1 100644
--- a/lua/crawler.lua
+++ b/lua/crawler.lua
@@ -7,7 +7,7 @@ local job = require('plenary.job')
 ---@field search_engine boolean
 local config = {
   render_markdown = true,
-  render_json = false,
+  render_json = true,
   search_engine = true,
 }
 
@@ -54,9 +54,9 @@ local function process_url(url, render_type)
 end
 
 local function insert_into_buffer(content)
-  local current_buf = vim.api.nvim_get_current_buf()
-  local current_line = vim.api.nvim_win_get_cursor(0)[1]
-  vim.api.nvim_buf_set_lines(current_buf, current_line, current_line, false, vim.split(content, '\n'))
+  local buf = vim.api.nvim_create_buf(false, true)
+  vim.api.nvim_buf_set_lines(buf, 0, -1, false, vim.split(content, '\n'))
+  vim.api.nvim_command('sbuffer ' .. buf)
 end
 
 local function process_sitemap(url)
@@ -66,14 +66,25 @@ end
 
 local function process_search(query)
   local search_url = 's.jina.ai/' .. vim.fn.shellescape(query)
-  local response = curl.get(search_url)
   
-  if response.status ~= 200 then
-    print("Error performing search: " .. query)
-    return
-  end
+  job:new({
+    command = "curl",
+    args = { search_url },
+    on_exit = function(j, return_val)
+      if return_val == 0 then
+        local result = table.concat(j:result(), "\n")
+        vim.schedule(function()
+          insert_into_buffer(result)
+        end)
+      else
+        vim.schedule(function()
+          print("Error performing search: " .. query)
+        end)
+      end
+    end,
+  }):start()
 
-  insert_into_buffer(response.body)
+  print("Search in progress. Results will be displayed in a new buffer when ready.")
 end
 
 local function get_input(prompt)