hard stop / soft stop for cutoff (#177) martingale base

research/vectorized_loader.ipynb

@@ -1,620 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import pyarrow\n",
-    "import numpy as np\n",
-    "from numba import jit\n",
-    "import v2realbot.utils.config_handler as cfh"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Další info k pokračování je zde https://blog.quantinsti.com/tick-tick-ohlc-data-pandas-tutorial/"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 38,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<class 'pandas.core.frame.DataFrame'>\n",
-      "DatetimeIndex: 190261 entries, 2024-04-22 13:30:00.267711+00:00 to 2024-04-22 19:59:59.987614+00:00\n",
-      "Data columns (total 6 columns):\n",
-      " #   Column      Non-Null Count   Dtype  \n",
-      "---  ------      --------------   -----  \n",
-      " 0   exchange    190261 non-null  object \n",
-      " 1   price       190261 non-null  float64\n",
-      " 2   size        190261 non-null  float64\n",
-      " 3   id          190261 non-null  int64  \n",
-      " 4   conditions  190261 non-null  object \n",
-      " 5   tape        190261 non-null  object \n",
-      "dtypes: float64(2), int64(1), object(3)\n",
-      "memory usage: 10.2+ MB\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>exchange</th>\n",
-       "      <th>price</th>\n",
-       "      <th>size</th>\n",
-       "      <th>id</th>\n",
-       "      <th>conditions</th>\n",
-       "      <th>tape</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>timestamp</th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.267711+00:00</th>\n",
-       "      <td>K</td>\n",
-       "      <td>36.890</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>52983525037630</td>\n",
-       "      <td>[ , F, I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.300501+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241117014</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.305439+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241117496</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.314520+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241118034</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.335201+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241121369</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.902614+00:00</th>\n",
-       "      <td>V</td>\n",
-       "      <td>37.750</td>\n",
-       "      <td>1100.0</td>\n",
-       "      <td>56480705310575</td>\n",
-       "      <td>[ ]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.977134+00:00</th>\n",
-       "      <td>N</td>\n",
-       "      <td>37.745</td>\n",
-       "      <td>300.0</td>\n",
-       "      <td>52983559963478</td>\n",
-       "      <td>[ ]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.977137+00:00</th>\n",
-       "      <td>N</td>\n",
-       "      <td>37.740</td>\n",
-       "      <td>7300.0</td>\n",
-       "      <td>52983559963696</td>\n",
-       "      <td>[ ]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.978626+00:00</th>\n",
-       "      <td>V</td>\n",
-       "      <td>37.750</td>\n",
-       "      <td>16.0</td>\n",
-       "      <td>56480706886228</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.987614+00:00</th>\n",
-       "      <td>N</td>\n",
-       "      <td>37.745</td>\n",
-       "      <td>30.0</td>\n",
-       "      <td>52983559963958</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>190261 rows × 6 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                 exchange   price    size              id  \\\n",
-       "timestamp                                                                   \n",
-       "2024-04-22 13:30:00.267711+00:00        K  36.890     5.0  52983525037630   \n",
-       "2024-04-22 13:30:00.300501+00:00        D  37.005     1.0  71675241117014   \n",
-       "2024-04-22 13:30:00.305439+00:00        D  37.005     1.0  71675241117496   \n",
-       "2024-04-22 13:30:00.314520+00:00        D  37.005     1.0  71675241118034   \n",
-       "2024-04-22 13:30:00.335201+00:00        D  37.005     1.0  71675241121369   \n",
-       "...                                   ...     ...     ...             ...   \n",
-       "2024-04-22 19:59:59.902614+00:00        V  37.750  1100.0  56480705310575   \n",
-       "2024-04-22 19:59:59.977134+00:00        N  37.745   300.0  52983559963478   \n",
-       "2024-04-22 19:59:59.977137+00:00        N  37.740  7300.0  52983559963696   \n",
-       "2024-04-22 19:59:59.978626+00:00        V  37.750    16.0  56480706886228   \n",
-       "2024-04-22 19:59:59.987614+00:00        N  37.745    30.0  52983559963958   \n",
-       "\n",
-       "                                 conditions tape  \n",
-       "timestamp                                         \n",
-       "2024-04-22 13:30:00.267711+00:00  [ , F, I]    A  \n",
-       "2024-04-22 13:30:00.300501+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.305439+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.314520+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.335201+00:00     [ , I]    A  \n",
-       "...                                     ...  ...  \n",
-       "2024-04-22 19:59:59.902614+00:00        [ ]    A  \n",
-       "2024-04-22 19:59:59.977134+00:00        [ ]    A  \n",
-       "2024-04-22 19:59:59.977137+00:00        [ ]    A  \n",
-       "2024-04-22 19:59:59.978626+00:00     [ , I]    A  \n",
-       "2024-04-22 19:59:59.987614+00:00     [ , I]    A  \n",
-       "\n",
-       "[190261 rows x 6 columns]"
-      ]
-     },
-     "execution_count": 38,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "tdf=pd.read_parquet('trades_bac.parquet',engine='pyarrow')\n",
-    "#print(df)\n",
-    "df = tdf.loc['BAC']\n",
-    "df.info()\n",
-    "df"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 39,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "@jit(nopython=True)\n",
-    "def ohlcv_bars(ticks, start_time, end_time, resolution):\n",
-    "    \"\"\"\n",
-    "    Generate OHLCV bars from tick data, skipping intervals without trading activity.\n",
-    "    \n",
-    "    Parameters:\n",
-    "    - ticks: numpy array with columns [timestamp, price, size]\n",
-    "    - start_time: the start timestamp for bars (Unix timestamp)\n",
-    "    - end_time: the end timestamp for bars (Unix timestamp)\n",
-    "    - resolution: time resolution in seconds\n",
-    "    \n",
-    "    Returns:\n",
-    "    - OHLCV bars as a numpy array\n",
-    "    \"\"\"\n",
-    "    num_bars = (end_time - start_time) // resolution + 1\n",
-    "    bar_list = []\n",
-    "\n",
-    "    for i in range(num_bars):\n",
-    "        bar_start_time = start_time + i * resolution\n",
-    "        bar_end_time = bar_start_time + resolution\n",
-    "        bar_ticks = ticks[(ticks[:, 0] >= bar_start_time) & (ticks[:, 0] < bar_end_time)]\n",
-    "        \n",
-    "        if bar_ticks.shape[0] == 0:\n",
-    "            continue  # Skip this bar as there are no ticks\n",
-    "\n",
-    "        # Calculate OHLCV values\n",
-    "        open_price = bar_ticks[0, 1]  # open\n",
-    "        high_price = np.max(bar_ticks[:, 1])  # high\n",
-    "        low_price = np.min(bar_ticks[:, 1])  # low\n",
-    "        close_price = bar_ticks[-1, 1]  # close\n",
-    "        volume = np.sum(bar_ticks[:, 2])  # volume\n",
-    "        bar_time = bar_start_time  # timestamp for the bar\n",
-    "\n",
-    "        bar_list.append([open_price, high_price, low_price, close_price, volume, bar_time])\n",
-    "\n",
-    "    # Convert list to numpy array\n",
-    "    if bar_list:\n",
-    "        ohlcv = np.array(bar_list)\n",
-    "    else:\n",
-    "        ohlcv = np.empty((0, 6))  # return an empty array if no bars were created\n",
-    "\n",
-    "    return ohlcv\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 40,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<class 'pandas.core.frame.DataFrame'>\n",
-      "DatetimeIndex: 190261 entries, 2024-04-22 13:30:00.267711+00:00 to 2024-04-22 19:59:59.987614+00:00\n",
-      "Data columns (total 6 columns):\n",
-      " #   Column      Non-Null Count   Dtype  \n",
-      "---  ------      --------------   -----  \n",
-      " 0   exchange    190261 non-null  object \n",
-      " 1   price       190261 non-null  float64\n",
-      " 2   size        190261 non-null  float64\n",
-      " 3   id          190261 non-null  int64  \n",
-      " 4   conditions  190261 non-null  object \n",
-      " 5   tape        190261 non-null  object \n",
-      "dtypes: float64(2), int64(1), object(3)\n",
-      "memory usage: 10.2+ MB\n"
-     ]
-    }
-   ],
-   "source": [
-    "df.info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 41,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['C', 'O', '4', 'B', '7', 'V', 'P', 'W', 'U', 'Z', 'F']\n",
-      "<class 'pandas.core.frame.DataFrame'>\n",
-      "DatetimeIndex: 143751 entries, 2024-04-22 13:30:00.300501+00:00 to 2024-04-22 19:59:59.987614+00:00\n",
-      "Data columns (total 6 columns):\n",
-      " #   Column      Non-Null Count   Dtype  \n",
-      "---  ------      --------------   -----  \n",
-      " 0   exchange    143751 non-null  object \n",
-      " 1   price       143751 non-null  float64\n",
-      " 2   size        143751 non-null  float64\n",
-      " 3   id          143751 non-null  int64  \n",
-      " 4   conditions  143751 non-null  object \n",
-      " 5   tape        143751 non-null  object \n",
-      "dtypes: float64(2), int64(1), object(3)\n",
-      "memory usage: 7.7+ MB\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>exchange</th>\n",
-       "      <th>price</th>\n",
-       "      <th>size</th>\n",
-       "      <th>id</th>\n",
-       "      <th>conditions</th>\n",
-       "      <th>tape</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>timestamp</th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.300501+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241117014</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.305439+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241117496</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.314520+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241118034</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.335201+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241121369</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 13:30:00.346219+00:00</th>\n",
-       "      <td>D</td>\n",
-       "      <td>37.005</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>71675241122389</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.902614+00:00</th>\n",
-       "      <td>V</td>\n",
-       "      <td>37.750</td>\n",
-       "      <td>1100.0</td>\n",
-       "      <td>56480705310575</td>\n",
-       "      <td>[ ]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.977134+00:00</th>\n",
-       "      <td>N</td>\n",
-       "      <td>37.745</td>\n",
-       "      <td>300.0</td>\n",
-       "      <td>52983559963478</td>\n",
-       "      <td>[ ]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.977137+00:00</th>\n",
-       "      <td>N</td>\n",
-       "      <td>37.740</td>\n",
-       "      <td>7300.0</td>\n",
-       "      <td>52983559963696</td>\n",
-       "      <td>[ ]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.978626+00:00</th>\n",
-       "      <td>V</td>\n",
-       "      <td>37.750</td>\n",
-       "      <td>16.0</td>\n",
-       "      <td>56480706886228</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2024-04-22 19:59:59.987614+00:00</th>\n",
-       "      <td>N</td>\n",
-       "      <td>37.745</td>\n",
-       "      <td>30.0</td>\n",
-       "      <td>52983559963958</td>\n",
-       "      <td>[ , I]</td>\n",
-       "      <td>A</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>143751 rows × 6 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                 exchange   price    size              id  \\\n",
-       "timestamp                                                                   \n",
-       "2024-04-22 13:30:00.300501+00:00        D  37.005     1.0  71675241117014   \n",
-       "2024-04-22 13:30:00.305439+00:00        D  37.005     1.0  71675241117496   \n",
-       "2024-04-22 13:30:00.314520+00:00        D  37.005     1.0  71675241118034   \n",
-       "2024-04-22 13:30:00.335201+00:00        D  37.005     1.0  71675241121369   \n",
-       "2024-04-22 13:30:00.346219+00:00        D  37.005     1.0  71675241122389   \n",
-       "...                                   ...     ...     ...             ...   \n",
-       "2024-04-22 19:59:59.902614+00:00        V  37.750  1100.0  56480705310575   \n",
-       "2024-04-22 19:59:59.977134+00:00        N  37.745   300.0  52983559963478   \n",
-       "2024-04-22 19:59:59.977137+00:00        N  37.740  7300.0  52983559963696   \n",
-       "2024-04-22 19:59:59.978626+00:00        V  37.750    16.0  56480706886228   \n",
-       "2024-04-22 19:59:59.987614+00:00        N  37.745    30.0  52983559963958   \n",
-       "\n",
-       "                                 conditions tape  \n",
-       "timestamp                                         \n",
-       "2024-04-22 13:30:00.300501+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.305439+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.314520+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.335201+00:00     [ , I]    A  \n",
-       "2024-04-22 13:30:00.346219+00:00     [ , I]    A  \n",
-       "...                                     ...  ...  \n",
-       "2024-04-22 19:59:59.902614+00:00        [ ]    A  \n",
-       "2024-04-22 19:59:59.977134+00:00        [ ]    A  \n",
-       "2024-04-22 19:59:59.977137+00:00        [ ]    A  \n",
-       "2024-04-22 19:59:59.978626+00:00     [ , I]    A  \n",
-       "2024-04-22 19:59:59.987614+00:00     [ , I]    A  \n",
-       "\n",
-       "[143751 rows x 6 columns]"
-      ]
-     },
-     "execution_count": 41,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "excludes = cfh.config_handler.get_val('AGG_EXCLUDED_TRADES')\n",
-    "print(excludes)\n",
-    "#excludes = [\"F\", \"I\"]\n",
-    "# FILTER EXCLUDED TRADES\n",
-    "# Filter rows to exclude those where 'conditions' contains 'F' or 'I'\n",
-    "# This simplifies the logic by directly using ~ (bitwise not operator) with np.isin\n",
-    "df = df[~df['conditions'].apply(lambda x: np.isin(x, excludes).any())]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 46,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/var/folders/8p/dwqnp65s0s77jdbm4_6z4vp80000gn/T/ipykernel_52602/3341929382.py:2: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
-      "  structured_array = np.array(list(zip(df.index, df['price'], df['size'])),\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[('2024-04-22T13:30:00.300501000', 37.005, 1.0e+00)\n",
-      " ('2024-04-22T13:30:00.305439000', 37.005, 1.0e+00)\n",
-      " ('2024-04-22T13:30:00.314520000', 37.005, 1.0e+00) ...\n",
-      " ('2024-04-22T19:59:59.977137000', 37.74 , 7.3e+03)\n",
-      " ('2024-04-22T19:59:59.978626000', 37.75 , 1.6e+01)\n",
-      " ('2024-04-22T19:59:59.987614000', 37.745, 3.0e+01)]\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "array([('2024-04-22T13:30:00.300501000', 37.005, 1.0e+00),\n",
-       "       ('2024-04-22T13:30:00.305439000', 37.005, 1.0e+00),\n",
-       "       ('2024-04-22T13:30:00.314520000', 37.005, 1.0e+00), ...,\n",
-       "       ('2024-04-22T19:59:59.977137000', 37.74 , 7.3e+03),\n",
-       "       ('2024-04-22T19:59:59.978626000', 37.75 , 1.6e+01),\n",
-       "       ('2024-04-22T19:59:59.987614000', 37.745, 3.0e+01)],\n",
-       "      dtype=[('timestamp', '<M8[ns]'), ('price', '<f8'), ('size', '<f8')])"
-      ]
-     },
-     "execution_count": 46,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# Creating a structured array with the timestamp as the first element\n",
-    "structured_array = np.array(list(zip(df.index, df['price'], df['size'])),\n",
-    "                            dtype=[('timestamp', 'datetime64[ns]'), ('price', 'float'), ('size', 'float')])\n",
-    "\n",
-    "print(structured_array)\n",
-    "structured_array\n",
-    "\n",
-    "# ticks = df[['index', 'price', 'size']].to_numpy()\n",
-    "# # ticks[:, 0] = pd.to_datetime(ticks[:, 0]).astype('int64') // 1_000_000_000  # \n",
-    "# ticks"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "resolution_seconds = 1  # 1 second resolution\n",
-    "ohlcv_data = ohlcv_bars(structured_array, resolution_seconds)\n",
-    "\n",
-    "# Converting the result back to DataFrame for better usability\n",
-    "ohlcv_df = pd.DataFrame(ohlcv_data, columns=['Open', 'High', 'Low', 'Close', 'Volume', 'Time'])\n",
-    "ohlcv_df['Time'] = pd.to_datetime(ohlcv_df['Time'], unit='s')  # Convert timestamps back to datetime\n"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": ".venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

restart.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # file: restart.sh
- 
+
 # Usage: ./restart.sh [test|prod|all]
 
 # Define server addresses

v2realbot/common/model.py

@@ -5,7 +5,7 @@ from rich import print
 from typing import Any, Optional, List, Union
 from datetime import datetime, date
 from pydantic import BaseModel, Field
-from v2realbot.enums.enums import Mode, Account, SchedulerStatus, Moddus, Market
+from v2realbot.enums.enums import Mode, Account, SchedulerStatus, Moddus
 from alpaca.data.enums import Exchange
 
 
@@ -159,7 +159,6 @@ class RunManagerRecord(BaseModel):
     mode: Mode
     note: Optional[str] = None
     ilog_save: bool = False
-    market: Optional[Market] = Market.US
     bt_from: Optional[datetime] = None
     bt_to: Optional[datetime] = None
     #weekdays filter

v2realbot/common/transform.py

@@ -5,7 +5,9 @@ import v2realbot.controller.services as cs
 
 #prevede dict radku zpatky na objekt vcetme retypizace
 def row_to_runmanager(row: dict) -> RunManagerRecord:
+
     is_running = cs.is_runner_running(row['runner_id']) if row['runner_id'] else False
+
     res = RunManagerRecord(
         moddus=row['moddus'],
         id=row['id'],
@@ -15,7 +17,6 @@ def row_to_runmanager(row: dict) -> RunManagerRecord:
         account=row['account'],
         note=row['note'],
         ilog_save=bool(row['ilog_save']),
-        market=row['market'] if row['market'] is not None else None,
         bt_from=datetime.fromisoformat(row['bt_from']) if row['bt_from'] else None,
         bt_to=datetime.fromisoformat(row['bt_to']) if row['bt_to'] else None,
         weekdays_filter=[int(x) for x in row['weekdays_filter'].split(',')] if row['weekdays_filter'] else [],

v2realbot/controller/run_manager.py

@@ -172,14 +172,14 @@ def add_run_manager_record(new_record: RunManagerRecord):
         # Construct a suitable INSERT query based on your RunManagerRecord fields
         insert_query = """
             INSERT INTO run_manager (moddus, id, strat_id, symbol,account, mode, note,ilog_save,
-                                    market, bt_from, bt_to, weekdays_filter, batch_id,
+                                    bt_from, bt_to, weekdays_filter, batch_id,
                                     start_time, stop_time, status, last_processed,
                                     history, valid_from, valid_to, testlist_id)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?) 
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 
         """
         values = [
             new_record.moddus, str(new_record.id), str(new_record.strat_id), new_record.symbol, new_record.account, new_record.mode, new_record.note,
-            int(new_record.ilog_save), new_record.market,
+            int(new_record.ilog_save),
             new_record.bt_from.isoformat() if new_record.bt_from is not None else None,
             new_record.bt_to.isoformat() if new_record.bt_to is not None else None,
             ",".join(str(x) for x in new_record.weekdays_filter) if new_record.weekdays_filter else None,

v2realbot/enums/enums.py

@@ -103,10 +103,4 @@ class StartBarAlign(str, Enum):
         RANDOM = first bar starts when first trade occurs
     """ 
     ROUND = "round"
-    RANDOM = "random"
+    RANDOM = "random"
-
-class Market(str, Enum):
-    US = "US"
-    CRYPTO = "CRYPTO"
-
-     

v2realbot/loader/aggregator_vectorized.py

@@ -1,122 +0,0 @@
-import pandas as pd
-import numpy as np
-from numba import jit
-from alpaca.data.historical import StockHistoricalDataClient
-from v2realbot.config import ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, DATA_DIR
-from alpaca.data.requests import StockTradesRequest
-import time
-from datetime import datetime
-from v2realbot.utils.utils import parse_alpaca_timestamp, ltp, zoneNY, send_to_telegram, fetch_calendar_data
-import pyarrow
-
-""""
-WIP - for later use
-
-"""""
-
-def fetch_stock_trades(symbol, start, end, max_retries=5, backoff_factor=1):
-    """
-    Attempts to fetch stock trades with exponential backoff. Raises an exception if all retries fail.
-
-    :param symbol: The stock symbol to fetch trades for.
-    :param start: The start time for the trade data.
-    :param end: The end time for the trade data.
-    :param max_retries: Maximum number of retries.
-    :param backoff_factor: Factor to determine the next sleep time.
-    :return: TradesResponse object.
-    :raises: ConnectionError if all retries fail.
-    """
-    client =  StockHistoricalDataClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY)
-    stockTradeRequest = StockTradesRequest(symbol_or_symbols=symbol, start=start, end=end)
-    last_exception = None
-
-    for attempt in range(max_retries):
-        try:
-            tradesResponse = client.get_stock_trades(stockTradeRequest)
-            print("Remote Fetch DAY DATA Complete", start, end)
-            return tradesResponse
-        except Exception as e:
-            print(f"Attempt {attempt + 1} failed: {e}")
-            last_exception = e
-            time.sleep(backoff_factor * (2 ** attempt))
-
-    print("All attempts to fetch data failed.")
-    raise ConnectionError(f"Failed to fetch stock trades after {max_retries} retries. Last exception: {str(last_exception)} and {format_exc()}")
-
-
-@jit(nopython=True)
-def ohlcv_bars(ticks, start_time, end_time, resolution):
-    """
-    Generate OHLCV bars from tick data, skipping intervals without trading activity.
-    
-    Parameters:
-    - ticks: numpy array with columns [timestamp, price, size]
-    - start_time: the start timestamp for bars (Unix timestamp)
-    - end_time: the end timestamp for bars (Unix timestamp)
-    - resolution: time resolution in seconds
-    
-    Returns:
-    - OHLCV bars as a numpy array
-    """
-    num_bars = (end_time - start_time) // resolution + 1
-    bar_list = []
-
-    for i in range(num_bars):
-        bar_start_time = start_time + i * resolution
-        bar_end_time = bar_start_time + resolution
-        bar_ticks = ticks[(ticks[:, 0] >= bar_start_time) & (ticks[:, 0] < bar_end_time)]
-        
-        if bar_ticks.shape[0] == 0:
-            continue  # Skip this bar as there are no ticks
-
-        # Calculate OHLCV values
-        open_price = bar_ticks[0, 1]  # open
-        high_price = np.max(bar_ticks[:, 1])  # high
-        low_price = np.min(bar_ticks[:, 1])  # low
-        close_price = bar_ticks[-1, 1]  # close
-        volume = np.sum(bar_ticks[:, 2])  # volume
-        bar_time = bar_start_time  # timestamp for the bar
-
-        bar_list.append([open_price, high_price, low_price, close_price, volume, bar_time])
-
-    # Convert list to numpy array
-    if bar_list:
-        ohlcv = np.array(bar_list)
-    else:
-        ohlcv = np.empty((0, 6))  # return an empty array if no bars were created
-
-    return ohlcv
-
-# Example usage
-if __name__ == '__main__':
-    # symbol = ["BAC"]
-    # #datetime in zoneNY 
-    # day_start = datetime(2024, 4, 22, 9, 30, 0)
-    # day_stop = datetime(2024, 4, 22, 16, 00, 0)
-
-    # day_start = zoneNY.localize(day_start)
-    # day_stop = zoneNY.localize(day_stop)
-
-    # tradesResponse = fetch_stock_trades(symbol, day_start, day_stop)
-
-    # df = tradesResponse.df
-    # df.to_parquet('trades_bac.parquet', engine='pyarrow')
-
-    df=pd.read_parquet('trades_bac.parquet',engine='pyarrow')
-    print(df)
-
-    #df = pd.read_csv('tick_data.csv')  # DF with tick data
-# Assuming 'df' is your DataFrame with columns 'time', 'price', 'size', 'condition'
-    exclude_conditions = ['ConditionA', 'ConditionB']  # Conditions to exclude
-    df_filtered = df[~df['condition'].isin(exclude_conditions)]
-    # Define your start and end times based on your trading session, ensure these are Unix timestamps
-    start_time = pd.to_datetime('2023-01-01 09:30:00').timestamp()
-    end_time = pd.to_datetime('2023-01-01 16:00:00').timestamp()
-    ticks = df[['time', 'price', 'size']].to_numpy()
-    ticks[:, 0] = pd.to_datetime(ticks[:, 0]).astype('int64') // 1_000_000_000  # Convert to Unix timestamp
-    resolution_seconds = 1  # 1 second resolution
-    ohlcv_data = ohlcv_bars(ticks, start_time, end_time, resolution_seconds)
-
-    # Converting the result back to DataFrame for better usability
-    ohlcv_df = pd.DataFrame(ohlcv_data, columns=['Open', 'High', 'Low', 'Close', 'Volume', 'Time'])
-    ohlcv_df['Time'] = pd.to_datetime(ohlcv_df['Time'], unit='s')  # Convert timestamps back to datetime

v2realbot/scheduler/ap_scheduler.py

@@ -2,7 +2,7 @@ from uuid import UUID
 from typing import Any, List, Tuple
 from uuid import UUID, uuid4
 from v2realbot.enums.enums import Moddus, SchedulerStatus, RecordType, StartBarAlign, Mode, Account, OrderSide
-from v2realbot.common.model import RunManagerRecord, StrategyInstance, RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest, Market
+from v2realbot.common.model import RunManagerRecord, StrategyInstance, RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest
 from v2realbot.utils.utils import validate_and_format_time, AttributeDict, zoneNY, zonePRG, safe_get, dict_replace_value, Store, parse_toml_string, json_serial, is_open_hours, send_to_telegram, concatenate_weekdays, transform_data
 from v2realbot.common.PrescribedTradeModel import Trade, TradeDirection, TradeStatus, TradeStoplossType
 from datetime import datetime
@@ -116,8 +116,7 @@ def initialize_jobs(run_manager_records: RunManagerRecord = None):
             scheduler.add_job(start_runman_record, start_trigger, id=f"scheduler_start_{record.id}", args=[record.id])
             scheduler.add_job(stop_runman_record, stop_trigger, id=f"scheduler_stop_{record.id}", args=[record.id])
                         
-    #scheduler.add_job(print_hello, 'interval', seconds=10, id=
+    #scheduler.add_job(print_hello, 'interval', seconds=10, id=f"scheduler_testinterval")
-    # f"scheduler_testinterval")
     scheduled_jobs = scheduler.get_jobs()
     print(f"APS jobs refreshed ({len(scheduled_jobs)})")
     current_jobs_dict = format_apscheduler_jobs(scheduled_jobs)
@@ -125,9 +124,9 @@ def initialize_jobs(run_manager_records: RunManagerRecord = None):
     return 0, current_jobs_dict
 
 #zastresovaci funkce resici error handling a printing
-def start_runman_record(id: UUID, debug_date = None):
+def start_runman_record(id: UUID, market = "US", debug_date = None):
     record = None
-    res, record, msg = _start_runman_record(id=id, debug_date=debug_date)
+    res, record, msg = _start_runman_record(id=id, market=market, debug_date=debug_date)
 
     if record is not None:
         market_time_now = datetime.now().astimezone(zoneNY) if debug_date is None else debug_date
@@ -166,8 +165,8 @@ def update_runman_record(record: RunManagerRecord):
         err_msg= f"STOP: Error updating {record.id} errir {set} with values {record}"
         return -2, err_msg#toto stopne zpracovani dalsich zaznamu pri chybe, zvazit continue
 
-def stop_runman_record(id: UUID, debug_date = None):
+def stop_runman_record(id: UUID, market = "US", debug_date = None):
-    res, record, msg = _stop_runman_record(id=id, debug_date=debug_date)
+    res, record, msg = _stop_runman_record(id=id, market=market, debug_date=debug_date)
     #results : 0 - ok, -1 not running/already running/not specific, -2 error
 
     #report vzdy zapiseme do history, pokud je record not None, pripadna chyba se stala po dotazeni recordu
@@ -197,7 +196,7 @@ def stop_runman_record(id: UUID, debug_date = None):
         print(f"STOP JOB: {id} FINISHED")
 
 #start function that is called from the job
-def _start_runman_record(id: UUID, debug_date = None):
+def _start_runman_record(id: UUID, market = "US", debug_date = None):
     print(f"Start scheduled record {id}")
 
     record : RunManagerRecord = None
@@ -208,16 +207,15 @@ def _start_runman_record(id: UUID, debug_date = None):
     
     record = result
 
-    if record.market == Market.US or record.market == Market.CRYPTO:
+    if market is not None and market == "US":
-        res, sada = sch.get_todays_market_times(market=record.market, debug_date=debug_date)
+        res, sada = sch.get_todays_market_times(market=market, debug_date=debug_date)
         if res == 0:
             market_time_now, market_open_datetime, market_close_datetime = sada
             print(f"OPEN:{market_open_datetime} CLOSE:{market_close_datetime}")
         else:
-            sada = f"Market {record.market} Error getting market times (CLOSED): " + str(sada)
+            sada = f"Market {market} Error getting market times (CLOSED): " + str(sada)
             return res, record, sada
-    else:
+    
-        print("Market type is unknown.")
     if cs.is_stratin_running(record.strat_id):
         return -1, record, f"Stratin {record.strat_id} is already running"
 
@@ -231,7 +229,7 @@ def _start_runman_record(id: UUID, debug_date = None):
     return 0, record, record.runner_id
 
 #stop function that is called from the job
-def _stop_runman_record(id: UUID, debug_date = None):
+def _stop_runman_record(id: UUID, market = "US", debug_date = None):
     record = None
     #get all records
     print(f"Stopping record {id}")
@@ -306,5 +304,5 @@ if __name__ == "__main__":
     # print(f"CALL FINISHED, with {debug_date} RESULT: {res}, {result}")
 
 
-    res, result = stop_runman_record(id=id, debug_date = debug_date)
+    res, result = stop_runman_record(id=id, market = "US", debug_date = debug_date)
     print(f"CALL FINISHED, with {debug_date} RESULT: {res}, {result}")

v2realbot/scheduler/scheduler.py

@@ -2,10 +2,10 @@ import json
 import datetime
 import v2realbot.controller.services as cs
 import v2realbot.controller.run_manager as rm
-from v2realbot.common.model import RunnerView, RunManagerRecord, StrategyInstance, Runner, RunRequest, Trade, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, Bar, RunArchiveChange, TestList, ConfigItem, InstantIndicator, DataTablesRequest, AnalyzerInputs, Market
+from v2realbot.common.model import RunnerView, RunManagerRecord, StrategyInstance, Runner, RunRequest, Trade, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, Bar, RunArchiveChange, TestList, ConfigItem, InstantIndicator, DataTablesRequest, AnalyzerInputs
 from uuid import uuid4, UUID
-from v2realbot.utils.utils import json_serial, send_to_telegram, zoneNY, zonePRG, zoneUTC, fetch_calendar_data
+from v2realbot.utils.utils import json_serial, send_to_telegram, zoneNY, zonePRG, fetch_calendar_data
-from datetime import datetime, timedelta, time
+from datetime import datetime, timedelta
 from traceback import format_exc
 from rich import print
 import requests
@@ -18,18 +18,9 @@ from v2realbot.config import WEB_API_KEY
 #naplanovany jako samostatni job a triggerován pouze jednou v daný čas pro start a stop 
 #novy kod v aps_scheduler.py
 
-def is_US_market_day(date):
+def get_todays_market_times(market = "US", debug_date = None):
-    cal_dates = fetch_calendar_data(date, date)
-    if len(cal_dates) == 0:
-        print("Today is not a market day.")
-        return False, cal_dates
-    else:
-        print("Market is open")
-        return True, cal_dates
-
-def get_todays_market_times(market, debug_date = None):
     try:
-        if market == Market.US:
+        if market == "US":
             #zjistit vsechny podminky - mozna loopovat - podminky jsou vlevo
             if debug_date is not None:
                 nowNY = debug_date
@@ -37,20 +28,17 @@ def get_todays_market_times(market, debug_date = None):
                 nowNY = datetime.now().astimezone(zoneNY)
             nowNY_date = nowNY.date()
             #is market open - nyni pouze US
-            stat, calendar_dates = is_US_market_day(nowNY_date)
+            cal_dates = fetch_calendar_data(nowNY_date, nowNY_date)
-            if stat:
+
+            if len(cal_dates) == 0:
+                print("No Market Day today")
+                return -1, "Market Closed"
             #zatim podpora pouze main session
+
             #pouze main session
-                market_open_datetime = zoneNY.localize(calendar_dates[0].open)
+            market_open_datetime = zoneNY.localize(cal_dates[0].open)
-                market_close_datetime = zoneNY.localize(calendar_dates[0].close)
+            market_close_datetime = zoneNY.localize(cal_dates[0].close)
-                return 0, (nowNY, market_open_datetime, market_close_datetime)
+            return 0, (nowNY, market_open_datetime, market_close_datetime)
-            else:
-                return -1, "Market is closed."
-        elif market == Market.CRYPTO:
-            now_market_datetime = datetime.now().astimezone(zoneUTC)
-            market_open_datetime = datetime.combine(datetime.now(), time.min)
-            matket_close_datetime = datetime.combine(datetime.now(), time.max)
-            return 0, (now_market_datetime, market_open_datetime, matket_close_datetime)
         else:
             return -1, "Market not supported"
     except Exception as e:

v2realbot/static/index.html

@@ -347,7 +347,6 @@
                                 <th>testlist_id</th>
                                 <th>Running</th>
                                 <th>RunnerId</th>
-                                <th>Market</th>
                             </tr>
                         </thead>
                         <tbody></tbody>

v2realbot/static/js/tables/runmanager/init.js

@@ -45,8 +45,7 @@ function initialize_runmanagerRecords() {
                         {data: 'valid_to', visible: true},
                         {data: 'testlist_id', visible: true},
                         {data: 'strat_running', visible: true},
-                        {data: 'runner_id', visible: true},
+                        {data: 'runner_id', visible: true},                  
-                        {data: 'market', visible: true},                  
                     ],
             paging: true,
             processing: true,