Adds You.com support

plugins/botchat/plugin.py

@@ -10,6 +10,7 @@ import logging
 import html2text
 import re
 import datetime
+import json
 
 logger=logging.getLogger("plugin.botchat")
 plugin_folder=os.path.dirname(os.path.realpath(__file__))
@@ -99,21 +100,81 @@ async def log_history(ctx, history):
 #            history_file.write(history)
     pass
 
+
+async def choose_random_searx_url():
+    """
+    Picks a random SearX URL to use
+
+    :param query: search query
+    """
+    instances_filename = os.path.join(plugin_folder, 'search_cache', "instances.json")
+    if (os.path.exists(instances_filename)):
+        logger.info(f"Using cached SearX URL list...")
+        with open(instances_filename, 'r') as instances_file:
+                 instance_json = json.loads(instances_file.read())
+                 instance_list = list(instance_json["instances"].keys())
+                 chosen_instance = instance_list[random.randint(0,len(instance_list)-1)]
+                 if "onion" in chosen_instance:
+                     return await choose_random_searx_url()
+                 return chosen_instance
+
+    async with aiohttp.ClientSession("https://searx.space") as session:
+        async with session.get("/data/instances.json", allow_redirects=True) as resp:
+            logger.info(f"SearX mirrors list response status {resp.status}")
+            if resp.status == 200:
+                response=await resp.json()
+                instance_list = list(response["instances"].keys())
+                chosen_instance = instance_list[random.randint(0,len(instance_list)-1)]
+                with open(instances_filename, 'w') as instances_file:
+                    instances_file.write(json.dumps(response))
+                return chosen_instance
+            else:
+                logger.info(f"Failed to check searx mirrors")
+                return llm_config["searx_url"]
+
+
 async def search_searx(query):
     """
     Searches the given query on SearX and returns an LLM summary
 
     :param query: search query
     """
-    search_url="https://metasearx.com/"
+    search_url=await choose_random_searx_url()
+    logger.info(f"Search URL: {search_url}")
     async with aiohttp.ClientSession(search_url) as session:
         search_params = { "q": query }
-        async with session.get("/", data=search_params) as resp:
+        async with session.get("/", allow_redirects=True, data=search_params) as resp:
             logger.info(f"Search response status {resp.status}")
-            response=await resp.text()
-            summary=await summarize(html2text.html2text(response))
-            logger.info(f"Search summary {summary}")
-            return summary
+            if resp.status == 200:
+                response=await resp.text()
+                summary=await summarize(html2text.html2text(response))
+                logger.info(f"Search summary {summary}")
+                return summary
+            else:
+                logger.info(f"Search failed... Retrying")
+                return await search_searx(query)
+
+async def search_you(query):
+    """
+    Searches the given query on You.com and returns an LLM summary
+
+    :param query: search query
+    """
+    search_url="https://api.ydc-index.io/search"
+    logger.info(f"Search URL: {search_url}")
+    async with aiohttp.ClientSession() as session:
+        search_params = { "query": query }
+        headers = {"X-API-Key": llm_config["you_token"]}
+        async with session.get(search_url, allow_redirects=True, params=search_params, headers=headers) as resp:
+            logger.info(f"Search response status {resp.status}")
+            if resp.status == 200:
+                response=await resp.text()
+                summary=await summarize(response)
+                logger.info(f"Search summary {summary}")
+                return summary
+            else:
+                logger.info(f"Search failed...")
+                return ""
 
 async def check_for_additional_context(chat_entries):
     #Check chat_entries
@@ -167,8 +228,9 @@ async def process_search(ctx, query_str):
     await ctx.channel.send(f"*Search [[#{search_id}]] processing...*")
     query_str_trimmed=query_str.strip()
     query=query_str_trimmed.removeprefix("/search")
-    search_results = await search_searx(query)
-    os.makedirs(os.path.join(plugin_folder, 'search_cache'))
+#    search_results = await search_searx(query)
+    search_results = await search_you(query)
+    os.makedirs(os.path.join(plugin_folder, 'search_cache'),exist_ok=True)
     search_filename = os.path.join(plugin_folder, 'search_cache', f"{search_id}.txt") 
     with open(search_filename, 'w') as search_file:
         search_file.write(search_results)