From ceaf3fbf78eb91054d161df29ea93628c6d74108 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Wed, 6 Mar 2024 15:38:25 +0200 Subject: [PATCH 01/33] py-llama predictor --- agent-arti/readme.md | 1 + agent-pyter/ccxt.py | 92 +++++++++ agent-pyter/lag-llama | 1 + agent-pyter/lag-llama.ipynb | 374 ++++++++++++++++++++++++++++++++++++ agent-pyter/notes.md | 1 + doc/oi-notes.md | 47 +++++ translate-nllb/nllb.py | 16 ++ 7 files changed, 532 insertions(+) create mode 100644 agent-arti/readme.md create mode 100644 agent-pyter/ccxt.py create mode 160000 agent-pyter/lag-llama create mode 100644 agent-pyter/lag-llama.ipynb create mode 100644 agent-pyter/notes.md create mode 100644 doc/oi-notes.md create mode 100644 translate-nllb/nllb.py diff --git a/agent-arti/readme.md b/agent-arti/readme.md new file mode 100644 index 0000000..7cb0311 --- /dev/null +++ b/agent-arti/readme.md @@ -0,0 +1 @@ +ToDo: copy arti code here \ No newline at end of file diff --git a/agent-pyter/ccxt.py b/agent-pyter/ccxt.py new file mode 100644 index 0000000..76c48d4 --- /dev/null +++ b/agent-pyter/ccxt.py @@ -0,0 +1,92 @@ +# # https://github.com/ccxt/ccxt/tree/master/examples/py +# # ! pip install ccxt +# # //cjs +# # var ccxt = require ('ccxt') +# # console.log (ccxt.exchanges) // print all available exchanges +# # py +# import ccxt +# #print(ccxt.exchanges) +# #import ccxt.async_support as ccxt + + +# # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +# # import ccxt + +# # hitbtc = ccxt.hitbtc({'verbose': True}) +# # bitmex = ccxt.bitmex() +# # huobipro = ccxt.huobipro() +# # exmo = ccxt.exmo({ +# # 'apiKey': 'YOUR_PUBLIC_API_KEY', +# # 'secret': 'YOUR_SECRET_PRIVATE_KEY', +# # }) +# # kraken = ccxt.kraken({ +# # 'apiKey': 'YOUR_PUBLIC_API_KEY', +# # 'secret': 'YOUR_SECRET_PRIVATE_KEY', +# # }) + +# # exchange_id = 'binance' +# # exchange_class = getattr(ccxt, exchange_id) +# # exchange = exchange_class({ +# # 'apiKey': 'YOUR_API_KEY', +# # 'secret': 'YOUR_SECRET', +# # }) + +# # hitbtc_markets = hitbtc.load_markets() + +# # print(hitbtc.id, hitbtc_markets) +# # print(bitmex.id, bitmex.load_markets()) +# # print(huobipro.id, huobipro.load_markets()) + +# # print(hitbtc.fetch_order_book(hitbtc.symbols[0])) +# # print(bitmex.fetch_ticker('BTC/USD')) +# # print(huobipro.fetch_trades('LTC/USDT')) + +# # print(exmo.fetch_balance()) + +# # # sell one ฿ for market price and receive $ right now +# # print(exmo.id, exmo.create_market_sell_order('BTC/USD', 1)) + +# # # limit buy BTC/EUR, you pay €2500 and receive ฿1 when the order is closed +# # print(exmo.id, exmo.create_limit_buy_order('BTC/EUR', 1, 2500.00)) + +# # # pass/redefine custom exchange-specific order params: type, amount, price, flags, etc... +# # kraken.create_market_buy_order('BTC/USD', 1, {'trading_agreement': 'agree'}) +# # # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +# # # -*- coding: utf-8 -*- + +# # import os +# # import sys +# # from pprint import pprint + +# # root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# # sys.path.append(root + '/python') + +# import ccxt # noqa: E402 + +# # ----------------------------------------------------------------------------- + +# # print('CCXT Version:', ccxt.__version__) + +# # ----------------------------------------------------------------------------- + +# exchange = ccxt.coinbase({ +# 'apiKey': 'tk2ShLJCmByejn78', +# 'secret': 'UcJfI5HzQmkEjclCeHFSfG8hnNYxaESv', +# # 'verbose': True, # for debug output +# }) + +# symbol = 'BTC/USDT' +# timeframe = '1m' +# since = None +# limit = None # not used by coinbase + +# try: +# # Max 300 Candles +# candles = exchange.fetch_ohlcv(symbol, timeframe, since, limit) +# pprint(candles) +# except Exception as err: +# print(err) + + diff --git a/agent-pyter/lag-llama b/agent-pyter/lag-llama new file mode 160000 index 0000000..9486655 --- /dev/null +++ b/agent-pyter/lag-llama @@ -0,0 +1 @@ +Subproject commit 948665530fcda634df9d7df0bee5e19b87785eb9 diff --git a/agent-pyter/lag-llama.ipynb b/agent-pyter/lag-llama.ipynb new file mode 100644 index 0000000..5e39217 --- /dev/null +++ b/agent-pyter/lag-llama.ipynb @@ -0,0 +1,374 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 15, + "id": "04e8bf3c-845f-49bb-9e9c-992d6b8948f0", + "metadata": {}, + "outputs": [], + "source": [ + "# https://colab.research.google.com/drive/1XxrLW9VGPlZDw3efTvUi0hQimgJOwQG6?usp=sharing#scrollTo=gyH5Xq9eSvzq" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "37f96736-8654-4852-a144-fd75df22aaf7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloning into 'lag-llama'...\n", + "remote: Enumerating objects: 124, done.\u001b[K\n", + "remote: Counting objects: 100% (69/69), done.\u001b[K\n", + "remote: Compressing objects: 100% (43/43), done.\u001b[K\n", + "remote: Total 124 (delta 39), reused 47 (delta 26), pack-reused 55\u001b[K\n", + "Receiving objects: 100% (124/124), 190.17 KiB | 2.29 MiB/s, done.\n", + "Resolving deltas: 100% (49/49), done.\n" + ] + } + ], + "source": [ + "!git clone https://github.com/time-series-foundation-models/lag-llama/" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "f5fac8fa-5ac8-4330-97e0-8a2f4237ba0f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/workspace/repos/git.d-popov.com/ai-kevin/agent-pyter/lag-llama/lag-llama\n" + ] + } + ], + "source": [ + "cd ./lag-llama" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "968625c9-00fd-4037-b97c-33dfc4758491", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: gluonts[torch] in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (0.14.4)\n", + "Requirement already satisfied: numpy==1.23.5 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (1.23.5)\n", + "Requirement already satisfied: torch>=2.0.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (2.2.1)\n", + "Requirement already satisfied: wandb in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (0.16.4)\n", + "Requirement already satisfied: scipy in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 5)) (1.12.0)\n", + "Requirement already satisfied: pandas==2.1.4 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 6)) (2.1.4)\n", + "Requirement already satisfied: huggingface_hub[cli] in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from -r requirements.txt (line 7)) (0.21.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from pandas==2.1.4->-r requirements.txt (line 6)) (2.9.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from pandas==2.1.4->-r requirements.txt (line 6)) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from pandas==2.1.4->-r requirements.txt (line 6)) (2024.1)\n", + "Requirement already satisfied: pydantic<3,>=1.7 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gluonts[torch]->-r requirements.txt (line 1)) (2.6.3)\n", + "Requirement already satisfied: tqdm~=4.23 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gluonts[torch]->-r requirements.txt (line 1)) (4.66.2)\n", + "Requirement already satisfied: toolz~=0.10 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gluonts[torch]->-r requirements.txt (line 1)) (0.12.1)\n", + "Requirement already satisfied: typing-extensions~=4.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gluonts[torch]->-r requirements.txt (line 1)) (4.8.0)\n", + "Requirement already satisfied: lightning<2.2,>=2.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gluonts[torch]->-r requirements.txt (line 1)) (2.1.4)\n", + "Requirement already satisfied: pytorch-lightning<2.2,>=2.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gluonts[torch]->-r requirements.txt (line 1)) (2.1.4)\n", + "Requirement already satisfied: filelock in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (3.13.1)\n", + "Requirement already satisfied: sympy in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (1.12)\n", + "Requirement already satisfied: networkx in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (3.1.2)\n", + "Requirement already satisfied: fsspec in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (2024.2.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (8.9.2.26)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (2.19.3)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (12.1.105)\n", + "Requirement already satisfied: triton==2.2.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from torch>=2.0.0->-r requirements.txt (line 3)) (2.2.0)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=2.0.0->-r requirements.txt (line 3)) (12.4.99)\n", + "Requirement already satisfied: Click!=8.0.0,>=7.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (8.1.7)\n", + "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (3.1.42)\n", + "Requirement already satisfied: requests<3,>=2.0.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (2.31.0)\n", + "Requirement already satisfied: psutil>=5.0.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (5.9.0)\n", + "Requirement already satisfied: sentry-sdk>=1.0.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (1.40.6)\n", + "Requirement already satisfied: docker-pycreds>=0.4.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (0.4.0)\n", + "Requirement already satisfied: PyYAML in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (6.0.1)\n", + "Requirement already satisfied: setproctitle in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (1.3.3)\n", + "Requirement already satisfied: setuptools in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (68.0.0)\n", + "Requirement already satisfied: appdirs>=1.4.3 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (1.4.4)\n", + "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from wandb->-r requirements.txt (line 4)) (4.25.3)\n", + "Requirement already satisfied: packaging>=20.9 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from huggingface_hub[cli]->-r requirements.txt (line 7)) (23.2)\n", + "Requirement already satisfied: InquirerPy==0.3.4 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from huggingface_hub[cli]->-r requirements.txt (line 7)) (0.3.4)\n", + "Requirement already satisfied: pfzy<0.4.0,>=0.3.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from InquirerPy==0.3.4->huggingface_hub[cli]->-r requirements.txt (line 7)) (0.3.4)\n", + "Requirement already satisfied: prompt-toolkit<4.0.0,>=3.0.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from InquirerPy==0.3.4->huggingface_hub[cli]->-r requirements.txt (line 7)) (3.0.42)\n", + "Requirement already satisfied: six>=1.4.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from docker-pycreds>=0.4.0->wandb->-r requirements.txt (line 4)) (1.16.0)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 4)) (4.0.11)\n", + "Requirement already satisfied: lightning-utilities<2.0,>=0.8.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from lightning<2.2,>=2.0->gluonts[torch]->-r requirements.txt (line 1)) (0.10.1)\n", + "Requirement already satisfied: torchmetrics<3.0,>=0.7.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from lightning<2.2,>=2.0->gluonts[torch]->-r requirements.txt (line 1)) (1.3.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from pydantic<3,>=1.7->gluonts[torch]->-r requirements.txt (line 1)) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.16.3 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from pydantic<3,>=1.7->gluonts[torch]->-r requirements.txt (line 1)) (2.16.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests<3,>=2.0.0->wandb->-r requirements.txt (line 4)) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests<3,>=2.0.0->wandb->-r requirements.txt (line 4)) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests<3,>=2.0.0->wandb->-r requirements.txt (line 4)) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests<3,>=2.0.0->wandb->-r requirements.txt (line 4)) (2023.7.22)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from jinja2->torch>=2.0.0->-r requirements.txt (line 3)) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from sympy->torch>=2.0.0->-r requirements.txt (line 3)) (1.3.0)\n", + "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from fsspec->torch>=2.0.0->-r requirements.txt (line 3)) (3.9.3)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 4)) (5.0.1)\n", + "Requirement already satisfied: wcwidth in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from prompt-toolkit<4.0.0,>=3.0.1->InquirerPy==0.3.4->huggingface_hub[cli]->-r requirements.txt (line 7)) (0.2.13)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch>=2.0.0->-r requirements.txt (line 3)) (1.3.1)\n", + "Requirement already satisfied: attrs>=17.3.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch>=2.0.0->-r requirements.txt (line 3)) (23.2.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch>=2.0.0->-r requirements.txt (line 3)) (1.4.1)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch>=2.0.0->-r requirements.txt (line 3)) (6.0.5)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch>=2.0.0->-r requirements.txt (line 3)) (1.9.4)\n" + ] + } + ], + "source": [ + "!pip install -r requirements.txt #--quiet # this could take some time # ignore the errors displayed by colab" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8f10c802-4ffa-40f7-bd62-14ff13fae03c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: requests in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from requests) (2023.7.22)\n", + "Requirement already satisfied: matplotlib in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (3.8.3)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (1.2.0)\n", + "Requirement already satisfied: cycler>=0.10 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (4.49.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (1.4.5)\n", + "Requirement already satisfied: numpy<2,>=1.21 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (1.23.5)\n", + "Requirement already satisfied: packaging>=20.0 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (23.2)\n", + "Requirement already satisfied: pillow>=8 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (10.0.1)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (3.1.2)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from matplotlib) (2.9.0)\n", + "Requirement already satisfied: six>=1.5 in /config/miniconda3/envs/pygame/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n" + ] + } + ], + "source": [ + "!pip install --upgrade requests\n", + "!pip install matplotlib\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "0a64aa15-1477-44bc-b772-a9342a5640c8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Consider using `hf_transfer` for faster downloads. This solution comes with some limitations. See https://huggingface.co/docs/huggingface_hub/hf_transfer for more details.\n", + "./lag-llama.ckpt\n" + ] + } + ], + "source": [ + "!huggingface-cli download time-series-foundation-models/Lag-Llama lag-llama.ckpt --local-dir ./" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a328c513-558f-45ca-b900-b669c4ef33ed", + "metadata": {}, + "outputs": [], + "source": [ + "from itertools import islice\n", + "\n", + "from matplotlib import pyplot as plt\n", + "import matplotlib.dates as mdates\n", + "\n", + "import torch\n", + "from gluonts.evaluation import make_evaluation_predictions, Evaluator\n", + "from gluonts.dataset.repository.datasets import get_dataset\n", + "\n", + "from gluonts.dataset.pandas import PandasDataset\n", + "import pandas as pd\n", + "\n", + "from lag_llama.gluon.estimator import LagLlamaEstimator" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "f098efb9-490c-46b7-9ea3-bea1f2871fa5", + "metadata": {}, + "outputs": [], + "source": [ + "def get_lag_llama_predictions(dataset, prediction_length, num_samples=100):\n", + " ckpt = torch.load(\"lag-llama.ckpt\", map_location=torch.device('cuda:0')) # Uses GPU since in this Colab we use a GPU.\n", + " estimator_args = ckpt[\"hyper_parameters\"][\"model_kwargs\"]\n", + "\n", + " estimator = LagLlamaEstimator(\n", + " ckpt_path=\"lag-llama.ckpt\",\n", + " prediction_length=prediction_length,\n", + " context_length=32, # Should not be changed; this is what the released Lag-Llama model was trained with\n", + "\n", + " # estimator args\n", + " input_size=estimator_args[\"input_size\"],\n", + " n_layer=estimator_args[\"n_layer\"],\n", + " n_embd_per_head=estimator_args[\"n_embd_per_head\"],\n", + " n_head=estimator_args[\"n_head\"],\n", + " scaling=estimator_args[\"scaling\"],\n", + " time_feat=estimator_args[\"time_feat\"],\n", + "\n", + " batch_size=1,\n", + " num_parallel_samples=100\n", + " )\n", + "\n", + " lightning_module = estimator.create_lightning_module()\n", + " transformation = estimator.create_transformation()\n", + " predictor = estimator.create_predictor(transformation, lightning_module)\n", + "\n", + " forecast_it, ts_it = make_evaluation_predictions(\n", + " dataset=dataset,\n", + " predictor=predictor,\n", + " num_samples=num_samples\n", + " )\n", + " forecasts = list(forecast_it)\n", + " tss = list(ts_it)\n", + "\n", + " return forecasts, tss" + ] + }, + { + "cell_type": "raw", + "id": "e7e6dd60-7c0c-483f-86d4-b2ba7c4104d3", + "metadata": {}, + "source": [ + "import pandas as pd\n", + "from gluonts.dataset.pandas import PandasDataset\n", + "\n", + "url = (\n", + " \"https://gist.githubusercontent.com/rsnirwan/a8b424085c9f44ef2598da74ce43e7a3/raw/b6fdef21fe1f654787fa0493846c546b7f9c4df2/ts_long.csv\"\n", + ")\n", + "df = pd.read_csv(url, index_col=0, parse_dates=True)\n", + "# Set numerical columns as float32\n", + "for col in df.columns:\n", + " # Check if column is not of string type\n", + " if df[col].dtype != 'object' and pd.api.types.is_string_dtype(df[col]) == False:\n", + " df[col] = df[col].astype('float32')\n", + "\n", + "# Create the Pandas\n", + "dataset = PandasDataset.from_long_dataframe(df, target=\"target\", item_id=\"item_id\")\n", + "\n", + "backtest_dataset = dataset\n", + "prediction_length = 24 # Define your prediction length. We use 24 here since the data is of hourly frequency\n", + "num_samples = 100 # number of samples sampled from the probability distribution for each timestep\n", + "forecasts, tss = get_lag_llama_predictions(backtest_dataset, prediction_length, num_samples)\n", + "len(forecasts)\n", + "forecasts[0].samples.shape\n", + "plt.figure(figsize=(20, 15))\n", + "date_formater = mdates.DateFormatter('%b, %d')\n", + "plt.rcParams.update({'font.size': 15})\n", + "\n", + "# Iterate through the first 9 series, and plot the predicted samples\n", + "for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):\n", + " ax = plt.subplot(3, 3, idx+1)\n", + "\n", + " plt.plot(ts[-4 * prediction_length:].to_timestamp(), label=\"target\", )\n", + " forecast.plot( color='g')\n", + " plt.xticks(rotation=60)\n", + " ax.xaxis.set_major_formatter(date_formater)\n", + " ax.set_title(forecast.item_id)\n", + "\n", + "plt.gcf().tight_layout()\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "raw", + "id": "74dc9a03-435e-40a5-bbda-4ddac9f6cfb9", + "metadata": {}, + "source": [ + "# Set numerical columns as float32\n", + "for col in df.columns:\n", + " # Check if column is not of string type\n", + " if df[col].dtype != 'object' and pd.api.types.is_string_dtype(df[col]) == False:\n", + " df[col] = df[col].astype('float32')\n", + "\n", + "# Create the Pandas\n", + "dataset = PandasDataset.from_long_dataframe(df, target=\"target\", item_id=\"item_id\")\n", + "\n", + "backtest_dataset = dataset\n", + "prediction_length = 24 # Define your prediction length. We use 24 here since the data is of hourly frequency\n", + "num_samples = 100 # number of samples sampled from the probability distribution for each timestep\n", + "forecasts, tss = get_lag_llama_predictions(backtest_dataset, prediction_length, num_samples)\n", + "len(forecasts)\n", + "forecasts[0].samples.shape\n", + "plt.figure(figsize=(20, 15))\n", + "date_formater = mdates.DateFormatter('%b, %d')\n", + "plt.rcParams.update({'font.size': 15})\n", + "\n", + "# Iterate through the first 9 series, and plot the predicted samples\n", + "for idx, (forecast, ts) in islice(enumerate(zip(forecasts, tss)), 9):\n", + " ax = plt.subplot(3, 3, idx+1)\n", + "\n", + " plt.plot(ts[-4 * prediction_length:].to_timestamp(), label=\"target\", )\n", + " forecast.plot( color='g')\n", + " plt.xticks(rotation=60)\n", + " ax.xaxis.set_major_formatter(date_formater)\n", + " ax.set_title(forecast.item_id)\n", + "\n", + "plt.gcf().tight_layout()\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8be08b6-0cfd-45b5-ac23-142e9f388049", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/agent-pyter/notes.md b/agent-pyter/notes.md new file mode 100644 index 0000000..bbdafa1 --- /dev/null +++ b/agent-pyter/notes.md @@ -0,0 +1 @@ +https://github.com/ccxt/ccxt/tree/master/examples/py/ \ No newline at end of file diff --git a/doc/oi-notes.md b/doc/oi-notes.md new file mode 100644 index 0000000..d630783 --- /dev/null +++ b/doc/oi-notes.md @@ -0,0 +1,47 @@ + + + + interpreter --api_base http://192.168.0.11:11434/v1/ + +interpreter --model "gpt-3.5-turbo" # mistral +interpreter --model "mistral" --api_base http://192.168.0.11:11434/v1/ + + Mac/Linux: 'export OPENAI_API_KEY=your-key-here', + Windows: 'setx OPENAI_API_KEY your-key-here' then restart terminal. +interpreter --local + +interpreter --api_base http://192.168.0.11:11434/v1 --api_key "" --model openai/local + + + +interpreter --api_base http://192.168.0.137:1234/v1 --api_key "" --model openai/local +192.168.0.137 + + +# Load a model, start the server, and run this example in your terminal +# Choose between streaming and non-streaming mode by setting the "stream" field + +curl http://192.168.0.11:11434/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "messages": [ + { "role": "system", "content": "Always answer in rhymes." }, + { "role": "user", "content": "Introduce yourself." } + ], + "temperature": 0.7, + "max_tokens": -1, + "stream": false +}' + + +curl http://192.168.0.137:1234/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "messages": [ + { "role": "system", "content": "Always answer in rhymes." }, + { "role": "user", "content": "Introduce yourself." } + ], + "temperature": 0.7, + "max_tokens": -1, + "stream": false +}' \ No newline at end of file diff --git a/translate-nllb/nllb.py b/translate-nllb/nllb.py new file mode 100644 index 0000000..1784066 --- /dev/null +++ b/translate-nllb/nllb.py @@ -0,0 +1,16 @@ +# conda activate transformers +# conda install pip +# pip install https://github.com/huggingface/transformers/archive/nllb.zip -q +# pip install transformers + +from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline + +model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M") +tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M") + +translator = pipeline('translation', model=model, tokenizer=tokenizer, src_lang="tam_Taml", tgt_lang='eng_Latn', max_length = 400) +translator("திஸ் ஐஸ் எ வெரி குட் மாடல் ") + +# -------------------------------------------- +https://colab.research.google.com/drive/1o9r0QbEQZ1tn4eBVv-wajAtgwi6Lp-ZJ?usp=sharing#scrollTo=q49lJjXd4Jct +https://colab.research.google.com/drive/1QEF0U9AaBeQdjLw7SyRty2ENAD4Qiiil#scrollTo=TFm232e77QRT \ No newline at end of file From 90f4537a4cbd3fe85114eebf5e794ea797384287 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Fri, 22 Mar 2024 17:38:43 +0200 Subject: [PATCH 02/33] generated agent-1 --- .env | 12 ++++- .vscode/tasks.json | 14 +++++- _notes/arti/ideas.md | 27 ----------- agent-a/.env | 3 ++ agent-a/.gitignore | 0 agent-a/README.md | 0 agent-a/requirements.txt | 0 agent-a/setup.sh | 31 ++++++++++++ agent-a/src/agent/__init__.py | 0 agent-a/src/agent/agent.py | 47 +++++++++++++++++++ agent-a/src/llm/__init__.py | 0 agent-a/src/tools/__init__.py | 0 agent-a/src/utils/__init__.py | 0 {_notes => doc/_notes}/arti/.excalidraw.svg | 0 doc/_notes/arti/ideas.md | 32 +++++++++++++ {_notes => doc/_notes}/arti/neo4j.cql | 0 .../_notes}/arti/wikidata/import.sh | 0 {home => doc/home}/Prompt.md | 0 {home => doc/home}/homeassistant.md | 0 {home => doc/home}/logs.log | 0 {home => doc/home}/prompt1.md | 0 memory1/intint.neo.py | 7 +++ memory1/models.py | 7 --- scripts/aider.sh | 12 +++++ 24 files changed, 155 insertions(+), 37 deletions(-) delete mode 100644 _notes/arti/ideas.md create mode 100644 agent-a/.env create mode 100644 agent-a/.gitignore create mode 100644 agent-a/README.md create mode 100644 agent-a/requirements.txt create mode 100644 agent-a/setup.sh create mode 100644 agent-a/src/agent/__init__.py create mode 100644 agent-a/src/agent/agent.py create mode 100644 agent-a/src/llm/__init__.py create mode 100644 agent-a/src/tools/__init__.py create mode 100644 agent-a/src/utils/__init__.py rename {_notes => doc/_notes}/arti/.excalidraw.svg (100%) create mode 100644 doc/_notes/arti/ideas.md rename {_notes => doc/_notes}/arti/neo4j.cql (100%) rename {_notes => doc/_notes}/arti/wikidata/import.sh (100%) rename {home => doc/home}/Prompt.md (100%) rename {home => doc/home}/homeassistant.md (100%) rename {home => doc/home}/logs.log (100%) rename {home => doc/home}/prompt1.md (100%) create mode 100644 scripts/aider.sh diff --git a/.env b/.env index aa301cf..722f994 100644 --- a/.env +++ b/.env @@ -7,4 +7,14 @@ TTS_BACKEND_URL3=http://192.168.0.10:9008/asr #gpu TTS_BACKEND_URL4=http://192.168.0.10:9009/asr #cpu 9008-gpu WS_URL=ws://localhost:8081 SERVER_PORT_WS=8081 -SERVER_PORT_HTTP=3005 \ No newline at end of file +SERVER_PORT_HTTP=3005 + +# aider +AIDER_MODEL= +AIDER_4= +AIDER_35TURBO= + +# OPENAI_API_KEY=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 +OPENAI_API_BASE=https://api.deepseek.com/v1 +OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a +AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 2de4d30..cb79000 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -74,7 +74,17 @@ // "kind": "build", // "isDefault": true // } - // } - + // }, + , + { + "label": "Activate Conda Env, Set ENV Variable, and Open Shell", + "type": "shell", + "command": "bash --init-file <(echo 'source ~/miniconda3/etc/profile.d/conda.sh && conda activate aider && export OPENAI_API_KEY=xxx && aider --no-auto-commits')", + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "new" + }, + } ] } \ No newline at end of file diff --git a/_notes/arti/ideas.md b/_notes/arti/ideas.md deleted file mode 100644 index bf92d46..0000000 --- a/_notes/arti/ideas.md +++ /dev/null @@ -1,27 +0,0 @@ -Key features & principles: - - modal/plug & play design - - Biomimicing based - - self inferencing loop - - Graph->LLM->Graph based logic (Self reflect) - - attention (Short term memory) - - generalized & contextuaized memory schema (memory is strongly context dependent and temporal) -LLM module -Graph module -Short term memory module -mid-term memory (history on the toppic) -graph powered long term memory with embedding storage for skills & AII ( interface on some of the layers) -separate text IOs" - - multi agent communication module/console/ - - internal state/context/mood/STM - - actions output - - - - GRAPH schema - - idea - - is child of - - Q: Brainstorm neo4j schema for biomimicing memory storage as neo4j graph database. It should be similar to the way humans store, retrieve and generalize knowledge - -Memory model: diff --git a/agent-a/.env b/agent-a/.env new file mode 100644 index 0000000..bfb4ede --- /dev/null +++ b/agent-a/.env @@ -0,0 +1,3 @@ + NEO4J_URI="bolt://192.168.0.10:7687" + NEO4J_USER="neo4j" + NEO4J_PASSWORD="lucas-bicycle-powder-stretch-ford-9492" \ No newline at end of file diff --git a/agent-a/.gitignore b/agent-a/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/agent-a/README.md b/agent-a/README.md new file mode 100644 index 0000000..e69de29 diff --git a/agent-a/requirements.txt b/agent-a/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/agent-a/setup.sh b/agent-a/setup.sh new file mode 100644 index 0000000..3bdcb03 --- /dev/null +++ b/agent-a/setup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Function to create directories +create_directories() { + mkdir -p ./{data/{raw,processed},notebooks,src/{agent,llm,tools,utils},tests/{agent,llm,tools,utils}} +} + +# Function to create files +create_files() { + touch ./{.gitignore,requirements.txt,README.md} + touch ./src/{agent,llm,tools,utils}/__init__.py + touch ./tests/{agent,llm,tools,utils}/test_{agent,llm,tool1,tool2,utils}.py +} + +# Function to initialize Git repository +initialize_git() { + echo "Do you want to initialize a Git repository? (y/n)" + read answer + if [ "$answer" == "y" ]; then + git init + echo "Git repository initialized." + cd .. + fi +} + +# Main script execution +create_directories +create_files +#initialize_git + +echo "Project setup complete." \ No newline at end of file diff --git a/agent-a/src/agent/__init__.py b/agent-a/src/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/agent-a/src/agent/agent.py b/agent-a/src/agent/agent.py new file mode 100644 index 0000000..9c42097 --- /dev/null +++ b/agent-a/src/agent/agent.py @@ -0,0 +1,47 @@ +# src/agent/agent.py + +class Agent: + def __init__(self): + self.tools = [] # Initialize an empty list to store tools + + def add_tool(self, tool): + # Add a tool to the agent's toolbox + self.tools.append(tool) + + def remove_tool(self, tool): + # Remove a tool from the agent's toolbox + if tool in self.tools: + self.tools.remove(tool) + + def use_tool(self, tool, *args, **kwargs): + # Use a tool with the agent + if tool in self.tools: + return tool.use(*args, **kwargs) + else: + return "Tool not found in agent's toolbox." + + def explore(self): + # Implement the logic for exploring new ideas + pass + + def organize(self): + # Implement the logic for organizing knowledge + pass + + def improve(self): + # Implement the logic for improving reasoning + pass + + def rest(self): + # Implement the logic for resting and updating knowledge + pass + +# Example usage +if __name__ == "__main__": + agent = Agent() + # Add tools to the agent's toolbox + # agent.add_tool(some_tool_instance) + + # Use a tool + # result = agent.use_tool(some_tool_instance, some_arguments) + # print(result) \ No newline at end of file diff --git a/agent-a/src/llm/__init__.py b/agent-a/src/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/agent-a/src/tools/__init__.py b/agent-a/src/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/agent-a/src/utils/__init__.py b/agent-a/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/_notes/arti/.excalidraw.svg b/doc/_notes/arti/.excalidraw.svg similarity index 100% rename from _notes/arti/.excalidraw.svg rename to doc/_notes/arti/.excalidraw.svg diff --git a/doc/_notes/arti/ideas.md b/doc/_notes/arti/ideas.md new file mode 100644 index 0000000..9d6b9ca --- /dev/null +++ b/doc/_notes/arti/ideas.md @@ -0,0 +1,32 @@ +Key features & principles: + - modal/plug & play design + - Biomimicing based + - self inferencing loop + - Graph->LLM->Graph based logic (Self reflect) + - attention (Short term memory) + - generalized & contextuaized memory schema (memory is strongly context dependent and temporal) +LLM module +Graph module +Short term memory module +mid-term memory (history on the toppic) +graph powered long term memory with embedding storage for skills & AII ( interface on some of the layers) +separate text IOs" + - multi agent communication module/console/ + - internal state/context/mood/STM + - actions output + + + + GRAPH schema + + idea + - is child of + + Q: Brainstorm neo4j schema for biomimicing memory storage as neo4j graph database. It should be similar to the way humans store, retrieve and generalize knowledge + +Memory model: + + +-- example prompt: +you are an AI living native on a linux host. you have all local resources and installed software at your disposal: root console access, git repo and the internet as resources. you have your own codebase and will brainstorm and find ideas how to improve your reasoning and knowledge of the world. . To avoid messing up your health, you will prefer to install new packages and test ideas inside docker or virtual envs. You can test your ideas in a docker containers and can experiment with different ways to pipleline and organise the thought process. you will also explain each approach and promote the best containers. A local neo4j database is your memory and knowledge source. Whenever you are not certain about a toppic, or need to refresh your knowledge, you will check online for updated information. If the problem concerns npm, pip or other package or an exiting repository on github or other repository server, you will first update your internal database with the latest readme and usage documentaion from the repository authors and later search stackoverflow to update your skills. periodically you will reorgnize and tidy your "mmory" making it more easily accessible and updating the embedding vectors to allow easy access to the needed context when there is a problem to solve. You will build a tools library over time that wil lgelp you solve more complex problems and keep it in your memory together with a list of task completed with each tool. you will experiment with different pipelines for different tasks, memorise them and generalize and optimize existing pipelines over time to build a solid knowhow. +let's create the project setup in python. our main file will be the agent, and it will call the llm and other tools (maybe langchain), explore, organize and improve when 'resting'. diff --git a/_notes/arti/neo4j.cql b/doc/_notes/arti/neo4j.cql similarity index 100% rename from _notes/arti/neo4j.cql rename to doc/_notes/arti/neo4j.cql diff --git a/_notes/arti/wikidata/import.sh b/doc/_notes/arti/wikidata/import.sh similarity index 100% rename from _notes/arti/wikidata/import.sh rename to doc/_notes/arti/wikidata/import.sh diff --git a/home/Prompt.md b/doc/home/Prompt.md similarity index 100% rename from home/Prompt.md rename to doc/home/Prompt.md diff --git a/home/homeassistant.md b/doc/home/homeassistant.md similarity index 100% rename from home/homeassistant.md rename to doc/home/homeassistant.md diff --git a/home/logs.log b/doc/home/logs.log similarity index 100% rename from home/logs.log rename to doc/home/logs.log diff --git a/home/prompt1.md b/doc/home/prompt1.md similarity index 100% rename from home/prompt1.md rename to doc/home/prompt1.md diff --git a/memory1/intint.neo.py b/memory1/intint.neo.py index a40720e..f545895 100644 --- a/memory1/intint.neo.py +++ b/memory1/intint.neo.py @@ -11,6 +11,8 @@ class Neo4jConnection: # Create the schema self.create_schema() + self.test_retrieval() + # Close the connection self.close() @@ -21,6 +23,11 @@ class Neo4jConnection: with self.driver.session() as session: session.write_transaction(self._create_constraints_and_indexes) + def test_retrieval(tx): + #run MATCH (n) RETURN n LIMIT 25 + result = tx.run("MATCH (n) RETURN n LIMIT 25;") + + @staticmethod def _create_constraints_and_indexes(tx): # Constraints and indexes for Person diff --git a/memory1/models.py b/memory1/models.py index 52f595c..6a3c1ed 100644 --- a/memory1/models.py +++ b/memory1/models.py @@ -1,10 +1,3 @@ -class Person: - def __init__(self, person_id, name, age): - self.person_id = person_id - self.name = name - self.age = age - - class Memory: def __init__(self, memory_id, content, timestamp, importance, relevance, associated_tags): self.memory_id = memory_id diff --git a/scripts/aider.sh b/scripts/aider.sh new file mode 100644 index 0000000..53e8e44 --- /dev/null +++ b/scripts/aider.sh @@ -0,0 +1,12 @@ +#!/bin/bash +source ~/miniconda3/etc/profile.d/conda.sh # Adjust the path as per your Conda installation +conda activate aider +export OPENAI_API_KEY=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 + + +# aider --no-auto-commits + +OPENAI_API_BASE=https://api.deepseek.com/v1 +OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a +AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat +aider --openai-api-base https://api.deepseek.com/v1 --openai-api-key sk-99df7736351f4536bd72cd64a416318a --model deepseek-coder \ No newline at end of file From e6fa6e4547425ad659bea7cd0062b4f5571ac04c Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Fri, 22 Mar 2024 19:28:44 +0200 Subject: [PATCH 03/33] Add notes for interpreter configuration with GROQ AI --- _doc/oi-notes.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 _doc/oi-notes.md diff --git a/_doc/oi-notes.md b/_doc/oi-notes.md new file mode 100644 index 0000000..5f72b29 --- /dev/null +++ b/_doc/oi-notes.md @@ -0,0 +1,49 @@ + + + + interpreter --api_base http://192.168.0.11:11434/v1/ + +interpreter --model "gpt-3.5-turbo" # mistral +interpreter --model "mistral" --api_base http://192.168.0.11:11434/v1/ + + + Mac/Linux: 'export OPENAI_API_KEY=your-key-here', + Windows: 'setx OPENAI_API_KEY your-key-here' then restart terminal. +interpreter --local + +interpreter --api_base http://192.168.0.11:11434/v1 --api_key "" --model openai/local +interpreter --api_base http://192.168.0.137:1234/v1 --api_key "" --model openai/local +192.168.0.137 + + +################################# GROQ +export OPENAI_API_KEY=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 +interpreter -y --api_base https://api.groq.com/openai/v1 --model gemma-7b-it ## mixtral-8x7b-32768 # gemma-7b-it # llama2-70b-4096 +## +# Load a model, start the server, and run this example in your terminal +# Choose between streaming and non-streaming mode by setting the "stream" field + +curl http://192.168.0.11:11434/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "messages": [ + { "role": "system", "content": "Always answer in rhymes." }, + { "role": "user", "content": "Introduce yourself." } + ], + "temperature": 0.7, + "max_tokens": -1, + "stream": false +}' + + +curl http://192.168.0.137:1234/v1/chat/completions \ +-H "Content-Type: application/json" \ +-d '{ + "messages": [ + { "role": "system", "content": "Always answer in rhymes." }, + { "role": "user", "content": "Introduce yourself." } + ], + "temperature": 0.7, + "max_tokens": -1, + "stream": false +}' \ No newline at end of file From 6d0a5c1064883402b4280a4bf57e0aaba3837037 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sat, 23 Mar 2024 01:07:12 +0200 Subject: [PATCH 04/33] reorganize --- {doc => _doc}/_notes/arti/.excalidraw.svg | 0 {doc => _doc}/_notes/arti/ideas.md | 0 {doc => _doc}/_notes/arti/neo4j.cql | 0 {doc => _doc}/_notes/arti/wikidata/import.sh | 0 {doc => _doc}/home/Prompt.md | 0 {doc => _doc}/home/homeassistant.md | 0 {doc => _doc}/home/logs.log | 0 {doc => _doc}/home/prompt1.md | 0 _doc/oi-notes.md | 4 +- {doc => _doc}/packages notes environments.md | 0 _doc/scripts/aider.sh | 34 ++++++++++++++ {doc => _doc}/tts notes.txt | 0 doc/oi-notes.md | 47 -------------------- scripts/aider.sh | 12 ----- 14 files changed, 36 insertions(+), 61 deletions(-) rename {doc => _doc}/_notes/arti/.excalidraw.svg (100%) rename {doc => _doc}/_notes/arti/ideas.md (100%) rename {doc => _doc}/_notes/arti/neo4j.cql (100%) rename {doc => _doc}/_notes/arti/wikidata/import.sh (100%) rename {doc => _doc}/home/Prompt.md (100%) rename {doc => _doc}/home/homeassistant.md (100%) rename {doc => _doc}/home/logs.log (100%) rename {doc => _doc}/home/prompt1.md (100%) rename {doc => _doc}/packages notes environments.md (100%) create mode 100644 _doc/scripts/aider.sh rename {doc => _doc}/tts notes.txt (100%) delete mode 100644 doc/oi-notes.md delete mode 100644 scripts/aider.sh diff --git a/doc/_notes/arti/.excalidraw.svg b/_doc/_notes/arti/.excalidraw.svg similarity index 100% rename from doc/_notes/arti/.excalidraw.svg rename to _doc/_notes/arti/.excalidraw.svg diff --git a/doc/_notes/arti/ideas.md b/_doc/_notes/arti/ideas.md similarity index 100% rename from doc/_notes/arti/ideas.md rename to _doc/_notes/arti/ideas.md diff --git a/doc/_notes/arti/neo4j.cql b/_doc/_notes/arti/neo4j.cql similarity index 100% rename from doc/_notes/arti/neo4j.cql rename to _doc/_notes/arti/neo4j.cql diff --git a/doc/_notes/arti/wikidata/import.sh b/_doc/_notes/arti/wikidata/import.sh similarity index 100% rename from doc/_notes/arti/wikidata/import.sh rename to _doc/_notes/arti/wikidata/import.sh diff --git a/doc/home/Prompt.md b/_doc/home/Prompt.md similarity index 100% rename from doc/home/Prompt.md rename to _doc/home/Prompt.md diff --git a/doc/home/homeassistant.md b/_doc/home/homeassistant.md similarity index 100% rename from doc/home/homeassistant.md rename to _doc/home/homeassistant.md diff --git a/doc/home/logs.log b/_doc/home/logs.log similarity index 100% rename from doc/home/logs.log rename to _doc/home/logs.log diff --git a/doc/home/prompt1.md b/_doc/home/prompt1.md similarity index 100% rename from doc/home/prompt1.md rename to _doc/home/prompt1.md diff --git a/_doc/oi-notes.md b/_doc/oi-notes.md index 5f72b29..6f4d4f7 100644 --- a/_doc/oi-notes.md +++ b/_doc/oi-notes.md @@ -16,9 +16,9 @@ interpreter --api_base http://192.168.0.137:1234/v1 --api_key "" --model openai/ 192.168.0.137 -################################# GROQ +# ################################ GROQ ########################## working export OPENAI_API_KEY=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 -interpreter -y --api_base https://api.groq.com/openai/v1 --model gemma-7b-it ## mixtral-8x7b-32768 # gemma-7b-it # llama2-70b-4096 +interpreter -y --api_base https://api.groq.com/openai/v1 --model llama2-70b-4096 ## mixtral-8x7b-32768 # gemma-7b-it # llama2-70b-4096 ## # Load a model, start the server, and run this example in your terminal # Choose between streaming and non-streaming mode by setting the "stream" field diff --git a/doc/packages notes environments.md b/_doc/packages notes environments.md similarity index 100% rename from doc/packages notes environments.md rename to _doc/packages notes environments.md diff --git a/_doc/scripts/aider.sh b/_doc/scripts/aider.sh new file mode 100644 index 0000000..7f7d0bc --- /dev/null +++ b/_doc/scripts/aider.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# python -m pip install git+https://github.com/paul-gauthier/aider.git + +source ~/miniconda3/etc/profile.d/conda.sh # Adjust the path as per your Conda installation +conda activate aider +export OPENAI_API_KEY=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 + + +# aider --no-auto-commits + +OPENAI_API_BASE=https://api.deepseek.com/v1 +OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a +AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat +aider --openai-api-base https://api.deepseek.com/v1 --openai-api-key sk-99df7736351f4536bd72cd64a416318a --model deepseek-coder +aider --openai-api-base 'https://api.groq.com/openai/v1' --openai-api-key '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' --model 'llama2-70b-4096' + +usage: aider [-h] [--openai-api-key OPENAI_API_KEY] [--model MODEL] [--skip-model-availability-check SKIP_MODEL_AVAILABILITY_CHECK] [--4] [--4turbo] [--35turbo] [--voice-language VOICE_LANGUAGE] + [--openai-api-base OPENAI_API_BASE] [--openai-api-type OPENAI_API_TYPE] [--openai-api-version OPENAI_API_VERSION] [--openai-api-deployment-id OPENAI_API_DEPLOYMENT_ID] + [--openai-organization-id OPENAI_ORGANIZATION_ID] [--openrouter] [--edit-format EDIT_FORMAT] [--map-tokens MAP_TOKENS] [--input-history-file INPUT_HISTORY_FILE] + [--chat-history-file CHAT_HISTORY_FILE] [--dark-mode] [--light-mode] [--pretty | --no-pretty] [--stream | --no-stream] [--user-input-color USER_INPUT_COLOR] + [--tool-output-color TOOL_OUTPUT_COLOR] [--tool-error-color TOOL_ERROR_COLOR] [--assistant-output-color ASSISTANT_OUTPUT_COLOR] [--code-theme CODE_THEME] [--show-diffs] + [--git | --no-git] [--gitignore | --no-gitignore] [--aiderignore AIDERIGNORE] [--auto-commits | --no-auto-commits] [--dirty-commits | --no-dirty-commits] [--dry-run | --no-dry-run] + [--commit] [--version] [--check-update] [--skip-check-update] [--apply FILE] [--yes] [-v] [--show-repo-map] [--message COMMAND] [--message-file MESSAGE_FILE] [--encoding ENCODING] + [-c CONFIG_FILE] + +export OPENAI_API_KEY=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 +interpreter -y --api_base https://api.groq.com/openai/v1 --model gemma-7b-it ## mixtral-8x7b-32768 # gemma-7b-it # llama2-70b-4096 + + +# Setup OpenRouter access +export OPENAI_API_KEY=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 +export OPENAI_API_BASE=https://api.groq.com/openai/v1 +# For example, run aider with Claude 3 Opus using the diff editing format +aider --model llama2-70b-4096 --edit-format diff diff --git a/doc/tts notes.txt b/_doc/tts notes.txt similarity index 100% rename from doc/tts notes.txt rename to _doc/tts notes.txt diff --git a/doc/oi-notes.md b/doc/oi-notes.md deleted file mode 100644 index d630783..0000000 --- a/doc/oi-notes.md +++ /dev/null @@ -1,47 +0,0 @@ - - - - interpreter --api_base http://192.168.0.11:11434/v1/ - -interpreter --model "gpt-3.5-turbo" # mistral -interpreter --model "mistral" --api_base http://192.168.0.11:11434/v1/ - - Mac/Linux: 'export OPENAI_API_KEY=your-key-here', - Windows: 'setx OPENAI_API_KEY your-key-here' then restart terminal. -interpreter --local - -interpreter --api_base http://192.168.0.11:11434/v1 --api_key "" --model openai/local - - - -interpreter --api_base http://192.168.0.137:1234/v1 --api_key "" --model openai/local -192.168.0.137 - - -# Load a model, start the server, and run this example in your terminal -# Choose between streaming and non-streaming mode by setting the "stream" field - -curl http://192.168.0.11:11434/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "messages": [ - { "role": "system", "content": "Always answer in rhymes." }, - { "role": "user", "content": "Introduce yourself." } - ], - "temperature": 0.7, - "max_tokens": -1, - "stream": false -}' - - -curl http://192.168.0.137:1234/v1/chat/completions \ --H "Content-Type: application/json" \ --d '{ - "messages": [ - { "role": "system", "content": "Always answer in rhymes." }, - { "role": "user", "content": "Introduce yourself." } - ], - "temperature": 0.7, - "max_tokens": -1, - "stream": false -}' \ No newline at end of file diff --git a/scripts/aider.sh b/scripts/aider.sh deleted file mode 100644 index 53e8e44..0000000 --- a/scripts/aider.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -source ~/miniconda3/etc/profile.d/conda.sh # Adjust the path as per your Conda installation -conda activate aider -export OPENAI_API_KEY=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 - - -# aider --no-auto-commits - -OPENAI_API_BASE=https://api.deepseek.com/v1 -OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a -AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat -aider --openai-api-base https://api.deepseek.com/v1 --openai-api-key sk-99df7736351f4536bd72cd64a416318a --model deepseek-coder \ No newline at end of file From b7cafceccb17ac1fb3cc90ba065d277e482a7f10 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sat, 23 Mar 2024 01:07:39 +0200 Subject: [PATCH 05/33] reorganize2 --- {doc => _doc}/_notes/arti/.excalidraw.svg | 0 {doc => _doc}/_notes/arti/ideas.md | 0 {doc => _doc}/_notes/arti/neo4j.cql | 0 {doc => _doc}/_notes/arti/wikidata/import.sh | 0 {doc => _doc}/home/Prompt.md | 0 {doc => _doc}/home/homeassistant.md | 0 {doc => _doc}/home/logs.log | 0 {doc => _doc}/home/prompt1.md | 0 {doc => _doc}/oi-notes.md | 0 {doc => _doc}/packages notes environments.md | 0 {doc => _doc}/tts notes.txt | 0 agent-b/main.py | 1 + 12 files changed, 1 insertion(+) rename {doc => _doc}/_notes/arti/.excalidraw.svg (100%) rename {doc => _doc}/_notes/arti/ideas.md (100%) rename {doc => _doc}/_notes/arti/neo4j.cql (100%) rename {doc => _doc}/_notes/arti/wikidata/import.sh (100%) rename {doc => _doc}/home/Prompt.md (100%) rename {doc => _doc}/home/homeassistant.md (100%) rename {doc => _doc}/home/logs.log (100%) rename {doc => _doc}/home/prompt1.md (100%) rename {doc => _doc}/oi-notes.md (100%) rename {doc => _doc}/packages notes environments.md (100%) rename {doc => _doc}/tts notes.txt (100%) create mode 100644 agent-b/main.py diff --git a/doc/_notes/arti/.excalidraw.svg b/_doc/_notes/arti/.excalidraw.svg similarity index 100% rename from doc/_notes/arti/.excalidraw.svg rename to _doc/_notes/arti/.excalidraw.svg diff --git a/doc/_notes/arti/ideas.md b/_doc/_notes/arti/ideas.md similarity index 100% rename from doc/_notes/arti/ideas.md rename to _doc/_notes/arti/ideas.md diff --git a/doc/_notes/arti/neo4j.cql b/_doc/_notes/arti/neo4j.cql similarity index 100% rename from doc/_notes/arti/neo4j.cql rename to _doc/_notes/arti/neo4j.cql diff --git a/doc/_notes/arti/wikidata/import.sh b/_doc/_notes/arti/wikidata/import.sh similarity index 100% rename from doc/_notes/arti/wikidata/import.sh rename to _doc/_notes/arti/wikidata/import.sh diff --git a/doc/home/Prompt.md b/_doc/home/Prompt.md similarity index 100% rename from doc/home/Prompt.md rename to _doc/home/Prompt.md diff --git a/doc/home/homeassistant.md b/_doc/home/homeassistant.md similarity index 100% rename from doc/home/homeassistant.md rename to _doc/home/homeassistant.md diff --git a/doc/home/logs.log b/_doc/home/logs.log similarity index 100% rename from doc/home/logs.log rename to _doc/home/logs.log diff --git a/doc/home/prompt1.md b/_doc/home/prompt1.md similarity index 100% rename from doc/home/prompt1.md rename to _doc/home/prompt1.md diff --git a/doc/oi-notes.md b/_doc/oi-notes.md similarity index 100% rename from doc/oi-notes.md rename to _doc/oi-notes.md diff --git a/doc/packages notes environments.md b/_doc/packages notes environments.md similarity index 100% rename from doc/packages notes environments.md rename to _doc/packages notes environments.md diff --git a/doc/tts notes.txt b/_doc/tts notes.txt similarity index 100% rename from doc/tts notes.txt rename to _doc/tts notes.txt diff --git a/agent-b/main.py b/agent-b/main.py new file mode 100644 index 0000000..af5626b --- /dev/null +++ b/agent-b/main.py @@ -0,0 +1 @@ +Hello, world! From 4c37d113cac1f0205294d199c9bb9ad9cf5da7ff Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 26 Mar 2024 23:13:00 +0200 Subject: [PATCH 06/33] text segmentation prompt --- _doc/_notes/PROMPTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 _doc/_notes/PROMPTS.md diff --git a/_doc/_notes/PROMPTS.md b/_doc/_notes/PROMPTS.md new file mode 100644 index 0000000..fcdaa54 --- /dev/null +++ b/_doc/_notes/PROMPTS.md @@ -0,0 +1,8 @@ +You are an expert in extracting new information from text sorting it out in existing categories or creating new categoty (We're using Neo4j as graph database). +This is multi-step process: +1. Divide the text into paragraph simmilar chunks that have the same toppic. Add a summary to the paragraph. +2. Create Memory object for each summary +3. Extract facts from each paragraph and add them as knowledge linked to the paragraph as separate memory object linked to the first one. Look into verbs, ajectives and nouns to extract the plain information from the text. If there is a source code, do not interpret it, but remember it as linked Memory as it is, while adding summary and link it to the main "Memory". +4. Assign possible category and scientific field labels to the information by adding them as tags. +This systematization and segmentation will allow you to remember the text and store it in your long-term memory as knowledge graph, while providing easy access later. Respond in json format with the extracted and restructured data. +Here's the source text: From e32aee98c04f6b86fe28811c4902355aea8ccd79 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 1 Apr 2024 15:34:50 +0300 Subject: [PATCH 07/33] lmstudio dockerfile --- lmstudio/dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 lmstudio/dockerfile diff --git a/lmstudio/dockerfile b/lmstudio/dockerfile new file mode 100644 index 0000000..116f4b2 --- /dev/null +++ b/lmstudio/dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:20.04 + +# Avoid prompts from apt during build +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + wget \ + x11vnc \ + xvfb \ + && rm -rf /var/lib/apt/lists/* + +# Create the /app directory to hold the application +WORKDIR /app + +# https://medium.com/@renswunnink/why-i-love-appimage-for-linux-distros-924769510ec5 +RUN wget -O lmstudio.AppImage "https://releases.lmstudio.ai/linux/0.2.18/beta/LM_Studio-0.2.18.AppImage" && \ + chmod u+x lmstudio.AppImage && \ + ./lmstudio.AppImage --appimage-extract && \ + rm lmstudio.AppImage && \ + mv squashfs-root lmstudio + +# Setup a virtual display environment with XVFB +ENV DISPLAY=:99 +RUN Xvfb :99 -screen 0 1024x768x16 & +EXPOSE 8080 + +# Run LM Studio (assumes LM Studio can be run headlessly or in a server mode) +CMD ["./lmstudio/AppRun"] + +# build: docker build -t llmstudio . +# run: docker run (-dit) -p 8980:8080 llmstudio From ac00af2d866d5d24fadc1a3ccc4f8c17968d565d Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 1 Apr 2024 16:46:40 +0300 Subject: [PATCH 08/33] lmstudio is running --- lmstudio/dockerfile | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/lmstudio/dockerfile b/lmstudio/dockerfile index 116f4b2..a0ad78d 100644 --- a/lmstudio/dockerfile +++ b/lmstudio/dockerfile @@ -1,3 +1,4 @@ +#FROM ollama/ollama FROM ubuntu:20.04 # Avoid prompts from apt during build @@ -12,6 +13,7 @@ RUN apt-get update && apt-get install -y \ # Create the /app directory to hold the application WORKDIR /app + # https://medium.com/@renswunnink/why-i-love-appimage-for-linux-distros-924769510ec5 RUN wget -O lmstudio.AppImage "https://releases.lmstudio.ai/linux/0.2.18/beta/LM_Studio-0.2.18.AppImage" && \ chmod u+x lmstudio.AppImage && \ @@ -19,13 +21,52 @@ RUN wget -O lmstudio.AppImage "https://releases.lmstudio.ai/linux/0.2.18/beta/LM rm lmstudio.AppImage && \ mv squashfs-root lmstudio +# or adddependencies on another layer +RUN apt-get update && apt-get install -y \ + dbus pciutils \ + libglib2.0-0 \ + libnss3 \ + libgbm1 \ + libxshmfence1 \ + libgl1-mesa-glx \ + libegl1-mesa \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libgtk-3-0 \ + libasound2 + # Setup a virtual display environment with XVFB -ENV DISPLAY=:99 RUN Xvfb :99 -screen 0 1024x768x16 & +ENV DISPLAY=:99 EXPOSE 8080 +RUN ln -s /app/lmstudio/lm-studio /lm-studio + +#? RUN chmod +x /app/lmstudio/AppRun +# Create a startup script to run Xvfb and your application +RUN echo '#!/bin/bash\n\ +Xvfb :99 -screen 0 1024x768x16 &\n\ +export DISPLAY=:99\n\ +# Your application start command goes here\n\ +exec /app/lmstudio/AppRun --no-sandbox\n\ +' > /start-app.sh && chmod +x /start-app.sh +CMD ["/start-app.sh"] + +#> apt-get update && apt-get install -y git x11vnc +#> git clone https://github.com/novnc/noVNC.git /opt/noVNC + + + # Run LM Studio (assumes LM Studio can be run headlessly or in a server mode) -CMD ["./lmstudio/AppRun"] +#CMD ["./lmstudio/AppRun"] +#CMD ["./lmstudio/AppRun", "--no-sandbox"] +#CMD ["/bin/bash"] # interactive shell + # build: docker build -t llmstudio . # run: docker run (-dit) -p 8980:8080 llmstudio +# docker build -t llmstudio . && docker run -it -p 8980:8080 llmstudio + +# docker run --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all -it llmstudio +# docker build -t llmstudio . && docker run -dit -p 8980:8080 --volume /var/run/dbus:/var/run/dbus llmstudio +# docker build -t llmstudio . && docker run -it -p 8980:8080 --volume /var/run/dbus:/var/run/dbus --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --security-opt apparmor=unconfined llmstudio From 431db945dd2b2424473da68301f629f3f7d6c3a4 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 2 Apr 2024 19:38:04 +0300 Subject: [PATCH 09/33] lmstudio changes --- lmstudio/dockerfile | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/lmstudio/dockerfile b/lmstudio/dockerfile index a0ad78d..2883a67 100644 --- a/lmstudio/dockerfile +++ b/lmstudio/dockerfile @@ -8,7 +8,11 @@ RUN apt-get update && apt-get install -y \ wget \ x11vnc \ xvfb \ - && rm -rf /var/lib/apt/lists/* + net-tools \ + git \ + python3 \ + python3-numpy \ + novnc # Create the /app directory to hold the application WORKDIR /app @@ -22,38 +26,48 @@ RUN wget -O lmstudio.AppImage "https://releases.lmstudio.ai/linux/0.2.18/beta/LM mv squashfs-root lmstudio # or adddependencies on another layer -RUN apt-get update && apt-get install -y \ - dbus pciutils \ +RUN apt-get update && apt-get install -y \ + dbus pciutils \ libglib2.0-0 \ libnss3 \ libgbm1 \ libxshmfence1 \ libgl1-mesa-glx \ libegl1-mesa \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libgtk-3-0 \ - libasound2 + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libgtk-3-0 \ + libasound2 \ + && rm -rf /var/lib/apt/lists/* # Setup a virtual display environment with XVFB RUN Xvfb :99 -screen 0 1024x768x16 & ENV DISPLAY=:99 EXPOSE 8080 +# Expose port 5980 for noVNC +EXPOSE 5980 RUN ln -s /app/lmstudio/lm-studio /lm-studio #? RUN chmod +x /app/lmstudio/AppRun # Create a startup script to run Xvfb and your application +# Create a startup script RUN echo '#!/bin/bash\n\ Xvfb :99 -screen 0 1024x768x16 &\n\ export DISPLAY=:99\n\ -# Your application start command goes here\n\ +# Start X11VNC\n\ +x11vnc -display :99 -nopw -listen localhost -xkb -forever &\n\ +# Start noVNC\n\ +/opt/noVNC/utils/launch.sh --vnc localhost:5900 --listen 5980\n\ +# Start the application\n\ exec /app/lmstudio/AppRun --no-sandbox\n\ ' > /start-app.sh && chmod +x /start-app.sh + CMD ["/start-app.sh"] #> apt-get update && apt-get install -y git x11vnc #> git clone https://github.com/novnc/noVNC.git /opt/noVNC +#? x11vnc -display :99 -nopw -listen localhost -xkb -forever & @@ -67,6 +81,7 @@ CMD ["/start-app.sh"] # run: docker run (-dit) -p 8980:8080 llmstudio # docker build -t llmstudio . && docker run -it -p 8980:8080 llmstudio +# cd /mnt/storage/DEV/workspace/repos/git.d-popov.com/ai-kevin/lmstudio/ # docker run --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all -it llmstudio # docker build -t llmstudio . && docker run -dit -p 8980:8080 --volume /var/run/dbus:/var/run/dbus llmstudio # docker build -t llmstudio . && docker run -it -p 8980:8080 --volume /var/run/dbus:/var/run/dbus --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --security-opt apparmor=unconfined llmstudio From fb96c68f6f44606849bb6e9e21a1912e88f68d20 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Thu, 4 Apr 2024 01:17:39 +0300 Subject: [PATCH 10/33] devika setup --- _containers/devika/setup.gist | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 _containers/devika/setup.gist diff --git a/_containers/devika/setup.gist b/_containers/devika/setup.gist new file mode 100644 index 0000000..4655c9b --- /dev/null +++ b/_containers/devika/setup.gist @@ -0,0 +1,22 @@ +git clone https://github.com/stitionai/devika.git +conda create -n devika python=3.10 +conda activate devika +which python +/config/miniconda3/envs/devika/bin/python -m pip install -r requirements.txt + +/ui#>? +playwright install --with-deps +npm install +bun run dev +open new terminal +conda activate devika +rename sample.config.toml +fill out config.toml + + +/ui#> + bun run preview --host --port 3000 + SET VITE_API_BASE_URL=https://api.dev.d-popov.com + bun run dev --host + + /#>python3 devika.py --debug \ No newline at end of file From 3ad33c96654fd5603e3e821aa3128586f012acac Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sun, 7 Apr 2024 17:26:58 +0300 Subject: [PATCH 11/33] devika test prompt --- _doc/_notes/PROMPTS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/_doc/_notes/PROMPTS.md b/_doc/_notes/PROMPTS.md index fcdaa54..3b7bf8f 100644 --- a/_doc/_notes/PROMPTS.md +++ b/_doc/_notes/PROMPTS.md @@ -6,3 +6,8 @@ This is multi-step process: 4. Assign possible category and scientific field labels to the information by adding them as tags. This systematization and segmentation will allow you to remember the text and store it in your long-term memory as knowledge graph, while providing easy access later. Respond in json format with the extracted and restructured data. Here's the source text: + + + +------------ +create a web app that has a button to start recording sends it to latest whisper llm for STT, translates it to target language and shows the result in realtime. it should be streaming text with minimal lag. \ No newline at end of file From 36a8c5a0c38ec49e21a72973c840268d56d5566b Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sun, 7 Apr 2024 20:20:36 +0300 Subject: [PATCH 12/33] misc settings, ymls and cmds --- _containers/devika/setup.gist | 13 ++++++- {lmstudio => _containers/lmstudio}/dockerfile | 4 +- _containers/opendevin/dockerfile | 39 +++++++++++++++++++ _containers/opendevin/start-services.sh | 12 ++++++ agent-py-bot/cmd.sh | 3 ++ config.json | 15 +++++++ memory1/intint.neo.py | 3 -- 7 files changed, 82 insertions(+), 7 deletions(-) rename {lmstudio => _containers/lmstudio}/dockerfile (98%) create mode 100644 _containers/opendevin/dockerfile create mode 100644 _containers/opendevin/start-services.sh create mode 100644 config.json diff --git a/_containers/devika/setup.gist b/_containers/devika/setup.gist index 4655c9b..7eba67a 100644 --- a/_containers/devika/setup.gist +++ b/_containers/devika/setup.gist @@ -13,10 +13,19 @@ conda activate devika rename sample.config.toml fill out config.toml - /ui#> bun run preview --host --port 3000 SET VITE_API_BASE_URL=https://api.dev.d-popov.com bun run dev --host - /#>python3 devika.py --debug \ No newline at end of file +# RUN: + /#>conda activate devika && python3 devika.py --debug + /#>conda activate node && npx bun run dev --host + + + + TOML: + + google search: + https://developers.google.com/custom-search/v1/introduction + https://programmablesearchengine.google.com/controlpanel/overview?cx=0382a4a0cfd6745b7 \ No newline at end of file diff --git a/lmstudio/dockerfile b/_containers/lmstudio/dockerfile similarity index 98% rename from lmstudio/dockerfile rename to _containers/lmstudio/dockerfile index 2883a67..ba472d6 100644 --- a/lmstudio/dockerfile +++ b/_containers/lmstudio/dockerfile @@ -1,5 +1,5 @@ -#FROM ollama/ollama -FROM ubuntu:20.04 +FROM ollama/ollama +# FROM ubuntu:20.04 # Avoid prompts from apt during build ARG DEBIAN_FRONTEND=noninteractive diff --git a/_containers/opendevin/dockerfile b/_containers/opendevin/dockerfile new file mode 100644 index 0000000..84157fa --- /dev/null +++ b/_containers/opendevin/dockerfile @@ -0,0 +1,39 @@ +# docker build -t opendevin . && docker run -d --name OpenDevin-1 -p 3050:3000 -p 3051:3001 opendevin + +# docker run --name OpenDevin-dev -it opendevin + +# Start with a base image that has both Python and Node.js +FROM nikolaik/python-nodejs:python3.11-nodejs14 + +# Install system dependencies required for the project +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Clone the latest version of OpenDevin +WORKDIR /opendevin +RUN git clone https://github.com/OpenDevin/OpenDevin.git . + +# Install Python dependencies +WORKDIR /opendevin/backend +RUN python -m pipenv requirements > requirements.txt && python -m pip install -r requirements.txt +RUN python -m pip install -r requirements.txt + +# Install Node.js dependencies +WORKDIR /opendevin/frontend +RUN npm install monaco-editor +RUN npm install + +# Build the frontend +RUN npm run build + +# Expose backend and frontend ports +EXPOSE 3000 3001 + +# Add a script to start both backend and frontend services +WORKDIR /opendevin +COPY start-services.sh /opendevin/start-services.sh +RUN chmod +x /opendevin/start-services.sh + +CMD ["/opendevin/start-services.sh"] diff --git a/_containers/opendevin/start-services.sh b/_containers/opendevin/start-services.sh new file mode 100644 index 0000000..81a298a --- /dev/null +++ b/_containers/opendevin/start-services.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Navigate to the backend directory and start the backend server in the background +cd /opendevin/backend +uvicorn opendevin.server.listen:app --port 3000 & + +# Serve the frontend from the build directory +cd /opendevin/frontend/build +npx serve -l 3001 & + +# Keep the container running +wait diff --git a/agent-py-bot/cmd.sh b/agent-py-bot/cmd.sh index f7f52aa..d19e8f2 100644 --- a/agent-py-bot/cmd.sh +++ b/agent-py-bot/cmd.sh @@ -30,4 +30,7 @@ in python, create an app that will search for a news about a specific topic on t +# devika +create new homepage for memecoin. look at https://donk.meme/ for inspiration about functionality. but the design should be novel. + diff --git a/config.json b/config.json new file mode 100644 index 0000000..9990c5b --- /dev/null +++ b/config.json @@ -0,0 +1,15 @@ +{ + "tabAutocompleteModel": { + "title": "Tab Autocomplete Model", + "provider": "ollama", + "model": "stable-code:code", + "apiBase": "https://ollama.d-popov.com" + } +} + + +// original: "tabAutocompleteModel": { +// "title": "Starcoder 3b", +// "provider": "ollama", +// "model": "starcoder-3b" +// }, \ No newline at end of file diff --git a/memory1/intint.neo.py b/memory1/intint.neo.py index f545895..eb5f358 100644 --- a/memory1/intint.neo.py +++ b/memory1/intint.neo.py @@ -30,9 +30,6 @@ class Neo4jConnection: @staticmethod def _create_constraints_and_indexes(tx): - # Constraints and indexes for Person - tx.run("CREATE CONSTRAINT ON (p:Person) ASSERT p.person_id IS UNIQUE;") - # Constraints and indexes for Memory tx.run("CREATE CONSTRAINT ON (m:Memory) ASSERT m.memory_id IS UNIQUE;") tx.run("CREATE INDEX ON :Memory(content);") From b56320570b5676c9f9bcbd2cc259796c3ceb6eb4 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sun, 7 Apr 2024 20:21:37 +0300 Subject: [PATCH 13/33] ignore rag --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ef81834..854e719 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ agent-py-bot/scrape/raw/* tts/*.m4a agent-mobile/jdk/* agent-mobile/artimobile/supervisord.pid +agent-pyter/lag-llama From eddf32a8043cdaa36b7af5ab031b69007c8612fd Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Thu, 18 Apr 2024 01:33:51 +0300 Subject: [PATCH 14/33] new dextools bot --- agent-pyter/dexbot.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 agent-pyter/dexbot.py diff --git a/agent-pyter/dexbot.py b/agent-pyter/dexbot.py new file mode 100644 index 0000000..fe5561b --- /dev/null +++ b/agent-pyter/dexbot.py @@ -0,0 +1,36 @@ +import requests +from bs4 import BeautifulSoup +import schedule +import time + +def check_new_pairs(): + url = "https://www.dextools.io/" + url = "https://www.dextools.io/app/en/bnb/pool-explorer" + response = requests.get(url) + soup = BeautifulSoup(response.text, 'html.parser') + + # Assuming the pairs are listed in
tags with a class that includes the word 'pair' + pairs = soup.find_all('div', class_=lambda x: x and 'pair' in x) + + current_pairs = {pair.text for pair in pairs} + + if not hasattr(check_new_pairs, "last_pairs"): + check_new_pairs.last_pairs = current_pairs + + new_pairs = current_pairs - check_new_pairs.last_pairs + if new_pairs: + print("New Pairs Found:", new_pairs) + # Here you can add the code to trigger any event (e.g., send an email, a notification, etc.) + + # Update the last checked pairs + check_new_pairs.last_pairs = current_pairs + +def main(): + schedule.every(10).seconds.do(check_new_pairs) + + while True: + schedule.run_pending() + time.sleep(1) + +if __name__ == "__main__": + main() From fa43c028f5cfc03fb5412e7f9f7c74d901e87395 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Thu, 18 Apr 2024 01:51:56 +0300 Subject: [PATCH 15/33] launch settings changed --- .vscode/launch.json | 57 ++++++++++++++++++++++++++------------------- .vscode/tasks.json | 44 +++++++++++++++++++--------------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d1eed27..ff9d9dd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,35 +1,27 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - { - "name": "Docker Node.js Launch", - "type": "docker", - "request": "launch", - "preLaunchTask": "docker-run: debug", - "platform": "node" - }, - { - "name": "Docker Python Launch?", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/agent-py-bot/agent.py", - "console": "integratedTerminal", - // "python": "${command:python.interpreterPath}", // Assumes Python extension is installed - // "preLaunchTask": "docker-run: python-debug", // You may need to create this task - // "env": { - // "PYTHONUNBUFFERED": "1" - // } - }, + // { + // "name": "Docker Node.js Launch", + // "type": "docker", + // "request": "launch", + // "preLaunchTask": "docker-run: debug", + // "platform": "node" + // }, + // { + // "name": "Docker Python Launch?", + // "type": "python", + // "request": "launch", + // "program": "${workspaceFolder}/agent-py-bot/agent.py", + // "console": "integratedTerminal" + // }, { "name": "Docker Python Launch with venv", "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/agent-py-bot/agent.py", "console": "integratedTerminal", - "python": "/venv/bin/python", // Path to the Python interpreter in your venv + "python": "/venv/bin/python", "env": { "PYTHONUNBUFFERED": "1" } @@ -42,7 +34,24 @@ "skipFiles": [ "/**" ] - + }, + { + "name": "Python Debugger: Python File", + "type": "debugpy", + "request": "launch", + "program": "${file}" + }, + { + "name": "Python Debugger: Python File with Conda", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "python": "${command:python.interpreterPath}", + "presentation": { + "clear": true + }, + "preLaunchTask": "conda-activate" // Name of your pre-launch task } ] } \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index cb79000..ad95bfe 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,24 +1,24 @@ { "version": "2.0.0", "tasks": [ - { - "type": "docker-build", - "label": "docker-build", - "platform": "node", - "dockerBuild": { - "dockerfile": "${workspaceFolder}/Dockerfile", - "context": "${workspaceFolder}", - "pull": true - } - }, - { - "type": "docker-run", - "label": "docker-run: release", - "dependsOn": [ - "docker-build" - ], - "platform": "node" - }, + // { + // "type": "docker-build", + // "label": "docker-build", + // "platform": "node", + // "dockerBuild": { + // "dockerfile": "${workspaceFolder}/Dockerfile", + // "context": "${workspaceFolder}", + // "pull": true + // } + // }, + // { + // "type": "docker-run", + // "label": "docker-run: release", + // "dependsOn": [ + // "docker-build" + // ], + // "platform": "node" + // }, // { // "type": "docker-run", // "label": "docker-run: debug2", @@ -85,6 +85,12 @@ "reveal": "always", "panel": "new" }, - } + }, + { + "label": "conda-activate", + "type": "shell", + "command": "conda activate py && echo 'Activated Conda Environment (py)!'", + "problemMatcher": [] + } ] } \ No newline at end of file From 1d455aaba20ceb4a6e9aaf1be4620aeda13c0060 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Thu, 18 Apr 2024 02:12:26 +0300 Subject: [PATCH 16/33] remove schedule dependency, as it is failing --- .vscode/tasks.json | 16 +++++++++++----- agent-pyter/dexbot.py | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ad95bfe..6a0c2a1 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -87,10 +87,16 @@ }, }, { - "label": "conda-activate", - "type": "shell", - "command": "conda activate py && echo 'Activated Conda Environment (py)!'", - "problemMatcher": [] - } + "label": "conda-activate", + "type": "shell", + "command": "source ~/miniconda3/etc/profile.d/conda.sh && conda activate py && echo 'Activated Conda Environment (py)!'", + "problemMatcher": [], + "options": { + "shell": { + "executable": "/bin/bash", + "args": ["-c"] + } + } + } ] } \ No newline at end of file diff --git a/agent-pyter/dexbot.py b/agent-pyter/dexbot.py index fe5561b..e90ef26 100644 --- a/agent-pyter/dexbot.py +++ b/agent-pyter/dexbot.py @@ -4,6 +4,7 @@ import schedule import time def check_new_pairs(): + print("Checking for new pairs...") url = "https://www.dextools.io/" url = "https://www.dextools.io/app/en/bnb/pool-explorer" response = requests.get(url) From e5f91cb4f8c4ef518149efa0b6a112e5b200145d Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sun, 12 May 2024 11:23:11 +0300 Subject: [PATCH 17/33] api urls env, confifs --- .env | 19 +++++++++++++------ .vscode/launch.json | 5 +++-- _doc/aider.md | 11 +++++++++++ _doc/oi-notes.md | 2 +- 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 _doc/aider.md diff --git a/.env b/.env index 722f994..3cac670 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ -TTS_BACKEND_URL=http://192.168.0.10:9008/asr +TTS_BACKEND_URL=http://192.168.0.10:9009/asr #TTS_BACKEND_URL=http://localhost:9001/asr #gpu 9002-cpu TTS_BACKEND_URL2=http://localhost:9002/asr TTS_BACKEND_URL3=http://192.168.0.10:9008/asr #gpu @@ -11,10 +11,17 @@ SERVER_PORT_HTTP=3005 # aider AIDER_MODEL= -AIDER_4= -AIDER_35TURBO= +AIDER_4=false +#AIDER_35TURBO= # OPENAI_API_KEY=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 -OPENAI_API_BASE=https://api.deepseek.com/v1 -OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a -AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat \ No newline at end of file +# OPENAI_API_BASE=https://api.deepseek.com/v1 +# OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a +# AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat + + +GROQ_API_KEY=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 +aider --model groq/llama3-70b-8192 + +# List models available from Groq +aider --models groq/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index ff9d9dd..2f20d3e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -47,11 +47,12 @@ "request": "launch", "program": "${file}", "console": "integratedTerminal", - "python": "${command:python.interpreterPath}", + //"python": "${command:python.interpreterPath}", + "python": "/config/miniconda3/envs/py/bin/python", "presentation": { "clear": true }, - "preLaunchTask": "conda-activate" // Name of your pre-launch task + //"preLaunchTask": "conda-activate" // Name of your pre-launch task } ] } \ No newline at end of file diff --git a/_doc/aider.md b/_doc/aider.md new file mode 100644 index 0000000..625a478 --- /dev/null +++ b/_doc/aider.md @@ -0,0 +1,11 @@ +python -m pip install git+https://github.com/d-popov/aider.git + + +export GROQ_API_KEY=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 +aider --models groq/ +aider --model groq/llama3-70b-8192 --no-auto-commits + + +#@ OLLAMA +export OPENAI_API_BASE=https://ollama.d-popov.com +aider --openai-api-base https://ollama.d-popov.com --openai-api-key ol-ddddd --models openai/ \ No newline at end of file diff --git a/_doc/oi-notes.md b/_doc/oi-notes.md index 6f4d4f7..5522572 100644 --- a/_doc/oi-notes.md +++ b/_doc/oi-notes.md @@ -15,7 +15,7 @@ interpreter --api_base http://192.168.0.11:11434/v1 --api_key "" --model openai/ interpreter --api_base http://192.168.0.137:1234/v1 --api_key "" --model openai/local 192.168.0.137 - +GROQ_API_KEY # ################################ GROQ ########################## working export OPENAI_API_KEY=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 interpreter -y --api_base https://api.groq.com/openai/v1 --model llama2-70b-4096 ## mixtral-8x7b-32768 # gemma-7b-it # llama2-70b-4096 From 7444b638e1c19aaf6a874690d9a64d670b71263a Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sun, 12 May 2024 11:24:31 +0300 Subject: [PATCH 18/33] dexbot --- .gitignore | 1 + agent-pyter/dexbot.py | 49 +++++++++++++++++++++++++++++++++++++---- agent-pyter/prompts.txt | 1 + 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 agent-pyter/prompts.txt diff --git a/.gitignore b/.gitignore index 854e719..25b2b34 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ tts/*.m4a agent-mobile/jdk/* agent-mobile/artimobile/supervisord.pid agent-pyter/lag-llama +agent-pyter/google-chrome-stable_current_amd64.deb diff --git a/agent-pyter/dexbot.py b/agent-pyter/dexbot.py index e90ef26..ff4a706 100644 --- a/agent-pyter/dexbot.py +++ b/agent-pyter/dexbot.py @@ -1,15 +1,55 @@ + +# source /path/to/virtualenv/bin/activate # On Unix or MacOS +# source /config/miniconda3/envs/py/bin/activate +# pip install requests beautifulsoup4 schedule selenium + +# wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +# sudo dpkg -i google-chrome-stable_current_amd64.deb +# apt install libnss3 libxss1 + import requests from bs4 import BeautifulSoup -import schedule +# import schedule +import time +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from webdriver_manager.chrome import ChromeDriverManager import time + +# Initialize WebDriver +service = Service(ChromeDriverManager().install()) +driver = webdriver.Chrome(service=service) + +def check_pairs_sel(): + + try: + # Open the page + driver.get("https://www.dextools.io/app/en/bnb/pool-explorer") + time.sleep(10) # Wait for JavaScript to execute + + # Extract the page source + html = driver.page_source + soup = BeautifulSoup(html, 'html.parser') + + # Assuming the pairs are listed in
tags with a class that includes the word 'pair' + pairs = soup.find_all('div', class_=lambda x: x and 'pair' in x) + print("Pairs found:", [pair.text for pair in pairs]) + finally: + driver.quit() + def check_new_pairs(): + # log the running time print("Checking for new pairs...") url = "https://www.dextools.io/" url = "https://www.dextools.io/app/en/bnb/pool-explorer" response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') + # make html dump to ./dextools-last.html + with open('./dextools-last.html', 'w') as f: + f.write(soup.prettify()) + # Assuming the pairs are listed in
tags with a class that includes the word 'pair' pairs = soup.find_all('div', class_=lambda x: x and 'pair' in x) @@ -27,11 +67,12 @@ def check_new_pairs(): check_new_pairs.last_pairs = current_pairs def main(): - schedule.every(10).seconds.do(check_new_pairs) + #schedule.every(10).seconds.do(check_new_pairs) while True: - schedule.run_pending() - time.sleep(1) + # schedule.run_pending() + check_pairs_sel() + time.sleep(10000) if __name__ == "__main__": main() diff --git a/agent-pyter/prompts.txt b/agent-pyter/prompts.txt new file mode 100644 index 0000000..d066faf --- /dev/null +++ b/agent-pyter/prompts.txt @@ -0,0 +1 @@ +create python app that will monitor for new pairs on https://dextools.io and trigger event immediately when there is new token \ No newline at end of file From 71f075dd63131619e22c68ade24d08f085d082f8 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sun, 12 May 2024 11:24:45 +0300 Subject: [PATCH 19/33] translation ui --- web/client.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/client.html b/web/client.html index 3ba9eb9..67ad2de 100644 --- a/web/client.html +++ b/web/client.html @@ -29,6 +29,10 @@ +