daily update

requirements.txt

@@ -84,7 +84,7 @@ jupyterlab_server==2.27.1
 jupyterlab_widgets==3.0.13
 kiwisolver==1.4.5
 korean-lunar-calendar==0.3.1
-lightweight_charts @ git+https://github.com/drew2323/lightweight-charts-python.git@35f029714b23c18abe791b90a85447e959c72258
+lightweight_charts @ git+https://github.com/drew2323/lightweight-charts-python.git@7986aa9195d9d3204a998d1a8f5778d95219a08e
 llvmlite==0.39.1
 lxml==5.3.0
 markdown-it-py==3.0.0
@@ -121,9 +121,11 @@ platformdirs==4.2.2
 plotly==5.24.0
 prometheus_client==0.21.0
 prompt_toolkit==3.0.47
+protobuf==5.28.2
 proxy-tools==0.1.0
 psutil==6.0.0
 psycopg2==2.9.9
+psycopg2-binary==2.9.9
 ptyprocess==0.7.0
 pure_eval==0.2.3
 pyarrow==11.0.0
@@ -147,6 +149,7 @@ PyWavelets==1.7.0
 pywebview==5.2
 PyYAML==6.0.2
 pyzmq==26.2.0
+ray==2.37.0
 referencing==0.35.1
 regex==2024.7.24
 requests==2.32.3
@@ -166,6 +169,7 @@ soupsieve==2.6
 SQLAlchemy==2.0.32
 sseclient-py==1.8.0
 stack-data==0.6.3
+stratlab_db @ git+https://gitea.stratlab.dev/Stratlab/db.git@0bbe486de7ac410a9375f2ccf7d557a658a662ea
 stumpy==1.13.0
 TA-Lib==0.4.32
 tenacity==9.0.0
@@ -193,7 +197,7 @@ webencodings==0.5.1
 websocket-client==1.8.0
 websockets==11.0.3
 Werkzeug==3.0.4
-widgetsnbextension==4.0.9
+widgetsnbextension==4.0.13
 yarl==1.13.1
 yfinance==0.2.43
 zipp==3.20.1

research/get_trades_at_once.ipynb

@@ -13,9 +13,38 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "trades_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n",
+      "trades_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n",
+      "trades_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n",
+      "trades_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
+      "ohlcv_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n",
+      "trades_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
+      "trades_df-BAC-2024-05-15T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
+      "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
+      "ohlcv_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n",
+      "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n",
+      "ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
+      "ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T15_30_00-47BCFOPUVWZ-100.parquet\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "import pandas as pd\n",
     "import numpy as np\n",
@@ -45,10 +74,10 @@
     "exclude_conditions = cfh.config_handler.get_val('AGG_EXCLUDED_TRADES') #standard ['C','O','4','B','7','V','P','W','U','Z','F']\n",
     "minsize = 100\n",
     "\n",
-    "symbol = \"SPY\"\n",
+    "symbol = \"BAC\"\n",
     "#datetime in zoneNY \n",
-    "day_start = datetime(2024, 1, 1, 9, 30, 0)\n",
+    "day_start = datetime(2023, 1, 1, 9, 30, 0)\n",
-    "day_stop = datetime(2024, 1, 14, 16, 00, 0)\n",
+    "day_stop = datetime(2024, 10, 2, 16, 00, 0)\n",
     "day_start = zoneNY.localize(day_start)\n",
     "day_stop = zoneNY.localize(day_stop)\n",
     "#filename of trades_df parquet, date are in isoformat but without time zone part\n",
@@ -56,13 +85,15 @@
     "#parquet interval cache contains exclude conditions and minsize filtering\n",
     "file_trades = dir + f\"trades_df-{symbol}-{day_start.strftime('%Y-%m-%dT%H:%M:%S')}-{day_stop.strftime('%Y-%m-%dT%H:%M:%S')}-{exclude_conditions}-{minsize}.parquet\"\n",
     "#file_trades = dir + f\"trades_df-{symbol}-{day_start.strftime('%Y-%m-%dT%H:%M:%S')}-{day_stop.strftime('%Y-%m-%dT%H:%M:%S')}.parquet\"\n",
-    "file_ohlcv = dir + f\"ohlcv_df-{symbol}-{day_start.strftime('%Y-%m-%dT%H:%M:%S')}-{day_stop.strftime('%Y-%m-%dT%H:%M:%S')}-{exclude_conditions}-{minsize}.parquet\"\n",
+    "file_ohlcv = dir + f\"ohlcv_df-{symbol}-{day_start.strftime('%Y-%m-%dT%H:%M:%S')}-{day_stop.strftime('%Y-%m-%dT%H:%M:%S')}-{str(exclude_conditions)}-{minsize}.parquet\"\n",
     "\n",
     "#PRINT all parquet in directory\n",
     "import os\n",
     "files = [f for f in os.listdir(dir) if f.endswith(\".parquet\")]\n",
     "for f in files:\n",
-    "    print(f)"
+    "    print(f)\n",
+    "\n",
+    "exclude_conditions"
    ]
   },
   {
@@ -71,13 +102,26 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "trades_df = fetch_daily_stock_trades(symbol, day_start, day_stop, exclude_conditions=exclude_conditions, minsize=minsize, force_remote=False, max_retries=5, backoff_factor=1)\n",
+    "#fetch trades in one go\n",
-    "trades_df"
+    "#trades_df = fetch_daily_stock_trades(symbol, day_start, day_stop, exclude_conditions=exclude_conditions, minsize=minsize, force_remote=False, max_retries=5, backoff_factor=1)\n",
+    "#fetch trades in parallel - for longer intervals\n",
+    "#trades_df = fetch_trades_parallel(symbol, day_start, day_stop, exclude_conditions=exclude_conditions, minsize=minsize, force_remote=False, max_workers=None)\n",
+    " \n",
+    "##trades_df.info()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#trades_df.to_parquet(file_trades, engine='pyarrow', compression='gzip')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [

research/strat_SUPERTREND/SUPERTREND_v1_SINGLE.ipynb

@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -80,7 +80,9 @@
       "trades_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n",
       "trades_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
       "ohlcv_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n",
+      "trades_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
       "trades_df-BAC-2024-05-15T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
+      "ohlcv_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
       "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
       "ohlcv_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n",
       "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n",
@@ -94,7 +96,7 @@
        "5"
       ]
      },
-     "execution_count": 2,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -129,36 +131,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
+     "ename": "NameError",
-     "output_type": "stream",
+     "evalue": "name 'basic_data' is not defined",
-     "text": [
+     "output_type": "error",
-      "<class 'pandas.core.frame.DataFrame'>\n",
+     "traceback": [
-      "DatetimeIndex: 57966 entries, 2024-02-12 09:30:00-05:00 to 2024-02-16 15:59:59-05:00\n",
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "Data columns (total 10 columns):\n",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      " #   Column      Non-Null Count  Dtype                     \n",
+      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mbasic_data\u001b[49m\u001b[38;5;241m.\u001b[39mdata[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBAC\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39minfo()\n\u001b[1;32m      3\u001b[0m df \u001b[38;5;241m=\u001b[39m basic_data\u001b[38;5;241m.\u001b[39mdata[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBAC\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m      5\u001b[0m nan_rows \u001b[38;5;241m=\u001b[39m df[df\u001b[38;5;241m.\u001b[39misna()\u001b[38;5;241m.\u001b[39many(axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)]\n",
-      "---  ------      --------------  -----                     \n",
+      "\u001b[0;31mNameError\u001b[0m: name 'basic_data' is not defined"
-      " 0   open        57966 non-null  float64                   \n",
-      " 1   high        57966 non-null  float64                   \n",
-      " 2   low         57966 non-null  float64                   \n",
-      " 3   close       57966 non-null  float64                   \n",
-      " 4   volume      57966 non-null  float64                   \n",
-      " 5   trades      57966 non-null  float64                   \n",
-      " 6   updated     57966 non-null  datetime64[ns, US/Eastern]\n",
-      " 7   vwap        57966 non-null  float64                   \n",
-      " 8   buyvolume   57966 non-null  float64                   \n",
-      " 9   sellvolume  57966 non-null  float64                   \n",
-      "dtypes: datetime64[ns, US/Eastern](1), float64(9)\n",
-      "memory usage: 4.9 MB\n"
      ]
     }
    ],
-   "source": [
+   "source": []
-    "basic_data.data[\"BAC\"].info()"
-   ]
   },
   {
    "cell_type": "markdown",

research/tests/ray_test.ipynb

@@ -0,0 +1,126 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-10-03 09:43:41,741\tINFO worker.py:1786 -- Started a local Ray instance.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "waiting for ray\n",
+      "ray returned all\n",
+      "ray finsihed 0.4031927839969285\n",
+      "worker 0 started\n",
+      "worker 0 finsihed\n",
+      "worker 1 started\n",
+      "worker 1 finsihed\n",
+      "worker 2 started\n",
+      "worker 2 finsihed\n",
+      "worker 3 started\n",
+      "worker 3 finsihed\n",
+      "serial function finsihed 0.21200023603159934\n",
+      "Ray with 4 parts: 0.4031927839969285 seconds\n",
+      "Serial: 0.21200023603159934 seconds\n",
+      "Serial computation is faster than Ray with 4 parts\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import timeit\n",
+    "import ray\n",
+    "\n",
+    "# Define the expensive function\n",
+    "@ray.remote\n",
+    "def expensive_function(n):\n",
+    "    # Generate a large random matrix\n",
+    "    A = np.random.rand(1000, 1000)\n",
+    "    B = np.random.rand(1000, 1000)\n",
+    "\n",
+    "    # Perform the matrix multiplication\n",
+    "    C = np.dot(A, B)\n",
+    "    # Return the result\n",
+    "    return C\n",
+    "\n",
+    "def expensive_function_serial(n):\n",
+    "    print(f\"worker {n} started\")\n",
+    "    # Generate a large random matrix\n",
+    "    A = np.random.rand(1000, 1000)\n",
+    "    B = np.random.rand(1000, 1000)\n",
+    "\n",
+    "    # Perform the matrix multiplication\n",
+    "    C = np.dot(A, B)\n",
+    "\n",
+    "    # Return the result\n",
+    "    print(f\"worker {n} finsihed\")\n",
+    "    return C\n",
+    "\n",
+    "# Initialize Ray\n",
+    "ray.init()\n",
+    "\n",
+    "# Create 4 remote actors to distribute the work\n",
+    "futures = [expensive_function.remote(_) for _ in range(4)]\n",
+    "\n",
+    "# Time the function using Ray with 4 parts\n",
+    "start_time = timeit.default_timer()\n",
+    "print(\"waiting for ray\")\n",
+    "results = ray.get(futures)\n",
+    "print(\"ray returned all\")\n",
+    "end_time = timeit.default_timer()\n",
+    "ray_time = end_time - start_time\n",
+    "print(\"ray finsihed\", ray_time)\n",
+    "\n",
+    "# Time the function serially\n",
+    "start_time = timeit.default_timer()\n",
+    "results = [expensive_function_serial(_) for _ in range(4)]\n",
+    "end_time = timeit.default_timer()\n",
+    "serial_time = end_time - start_time\n",
+    "print(\"serial function finsihed\", serial_time)\n",
+    "\n",
+    "\n",
+    "# Print the results\n",
+    "print(f\"Ray with 4 parts: {ray_time} seconds\")\n",
+    "print(f\"Serial: {serial_time} seconds\")\n",
+    "\n",
+    "# Compare the results\n",
+    "if ray_time < serial_time:\n",
+    "    print(\"Ray with 4 parts is faster than serial computation\")\n",
+    "else:\n",
+    "    print(\"Serial computation is faster than Ray with 4 parts\")\n",
+    "\n",
+    "# Shutdown Ray\n",
+    "ray.shutdown()\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}