sitemap added

* RunManagerRecord class has a new attribute market. Market enum is imported.

* row_to_runmanager function considers market column

* add_run_manager_record and update_run_manager_record functions are changed. fetch_all_markets_in_run_manager is new.

* new Market enumeration class is defined

* market_value used for job scheduling. start and stop functions have modifications of market parameter input

* new is_market_day function + modifications of get_todays_market_times function

* market attribute set default to US

* row_to_runmanager function has no string formatter for market attribute

* add_run_manager_record function adn update_run_manager_record function update the DB column market based on record.market data

* start_runman_record and stop_runman_record have got no market parameter

* get_todays_market_times function is changed

* default value for market atribute is Market.US

* update_run_manager_record function has no if condition for market key

* market_value deleted, used enumaration value Market.US instead of string US

* get_todays_market_times has a new if condition for Market.CRYPTO

* update includes market column in the run_manager table

* market attribute in Run Manager record has value given by enumeration as Market.US

* documentation of changes made in the branch

* remove README_feature_market.md

* back to original state

* Delete README_feature_market.md

* _start_runman_record has an additional else condition

* is_market_day renamed to is_US_market_day

* transferables column added into runner_header table

research/vectorized_loader.ipynb

@@ -0,0 +1,620 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import pyarrow\n",
+    "import numpy as np\n",
+    "from numba import jit\n",
+    "import v2realbot.utils.config_handler as cfh"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Další info k pokračování je zde https://blog.quantinsti.com/tick-tick-ohlc-data-pandas-tutorial/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "DatetimeIndex: 190261 entries, 2024-04-22 13:30:00.267711+00:00 to 2024-04-22 19:59:59.987614+00:00\n",
+      "Data columns (total 6 columns):\n",
+      " #   Column      Non-Null Count   Dtype  \n",
+      "---  ------      --------------   -----  \n",
+      " 0   exchange    190261 non-null  object \n",
+      " 1   price       190261 non-null  float64\n",
+      " 2   size        190261 non-null  float64\n",
+      " 3   id          190261 non-null  int64  \n",
+      " 4   conditions  190261 non-null  object \n",
+      " 5   tape        190261 non-null  object \n",
+      "dtypes: float64(2), int64(1), object(3)\n",
+      "memory usage: 10.2+ MB\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exchange</th>\n",
+       "      <th>price</th>\n",
+       "      <th>size</th>\n",
+       "      <th>id</th>\n",
+       "      <th>conditions</th>\n",
+       "      <th>tape</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>timestamp</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.267711+00:00</th>\n",
+       "      <td>K</td>\n",
+       "      <td>36.890</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>52983525037630</td>\n",
+       "      <td>[ , F, I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.300501+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241117014</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.305439+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241117496</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.314520+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241118034</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.335201+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241121369</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.902614+00:00</th>\n",
+       "      <td>V</td>\n",
+       "      <td>37.750</td>\n",
+       "      <td>1100.0</td>\n",
+       "      <td>56480705310575</td>\n",
+       "      <td>[ ]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.977134+00:00</th>\n",
+       "      <td>N</td>\n",
+       "      <td>37.745</td>\n",
+       "      <td>300.0</td>\n",
+       "      <td>52983559963478</td>\n",
+       "      <td>[ ]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.977137+00:00</th>\n",
+       "      <td>N</td>\n",
+       "      <td>37.740</td>\n",
+       "      <td>7300.0</td>\n",
+       "      <td>52983559963696</td>\n",
+       "      <td>[ ]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.978626+00:00</th>\n",
+       "      <td>V</td>\n",
+       "      <td>37.750</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>56480706886228</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.987614+00:00</th>\n",
+       "      <td>N</td>\n",
+       "      <td>37.745</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>52983559963958</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>190261 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                 exchange   price    size              id  \\\n",
+       "timestamp                                                                   \n",
+       "2024-04-22 13:30:00.267711+00:00        K  36.890     5.0  52983525037630   \n",
+       "2024-04-22 13:30:00.300501+00:00        D  37.005     1.0  71675241117014   \n",
+       "2024-04-22 13:30:00.305439+00:00        D  37.005     1.0  71675241117496   \n",
+       "2024-04-22 13:30:00.314520+00:00        D  37.005     1.0  71675241118034   \n",
+       "2024-04-22 13:30:00.335201+00:00        D  37.005     1.0  71675241121369   \n",
+       "...                                   ...     ...     ...             ...   \n",
+       "2024-04-22 19:59:59.902614+00:00        V  37.750  1100.0  56480705310575   \n",
+       "2024-04-22 19:59:59.977134+00:00        N  37.745   300.0  52983559963478   \n",
+       "2024-04-22 19:59:59.977137+00:00        N  37.740  7300.0  52983559963696   \n",
+       "2024-04-22 19:59:59.978626+00:00        V  37.750    16.0  56480706886228   \n",
+       "2024-04-22 19:59:59.987614+00:00        N  37.745    30.0  52983559963958   \n",
+       "\n",
+       "                                 conditions tape  \n",
+       "timestamp                                         \n",
+       "2024-04-22 13:30:00.267711+00:00  [ , F, I]    A  \n",
+       "2024-04-22 13:30:00.300501+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.305439+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.314520+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.335201+00:00     [ , I]    A  \n",
+       "...                                     ...  ...  \n",
+       "2024-04-22 19:59:59.902614+00:00        [ ]    A  \n",
+       "2024-04-22 19:59:59.977134+00:00        [ ]    A  \n",
+       "2024-04-22 19:59:59.977137+00:00        [ ]    A  \n",
+       "2024-04-22 19:59:59.978626+00:00     [ , I]    A  \n",
+       "2024-04-22 19:59:59.987614+00:00     [ , I]    A  \n",
+       "\n",
+       "[190261 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tdf=pd.read_parquet('trades_bac.parquet',engine='pyarrow')\n",
+    "#print(df)\n",
+    "df = tdf.loc['BAC']\n",
+    "df.info()\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@jit(nopython=True)\n",
+    "def ohlcv_bars(ticks, start_time, end_time, resolution):\n",
+    "    \"\"\"\n",
+    "    Generate OHLCV bars from tick data, skipping intervals without trading activity.\n",
+    "    \n",
+    "    Parameters:\n",
+    "    - ticks: numpy array with columns [timestamp, price, size]\n",
+    "    - start_time: the start timestamp for bars (Unix timestamp)\n",
+    "    - end_time: the end timestamp for bars (Unix timestamp)\n",
+    "    - resolution: time resolution in seconds\n",
+    "    \n",
+    "    Returns:\n",
+    "    - OHLCV bars as a numpy array\n",
+    "    \"\"\"\n",
+    "    num_bars = (end_time - start_time) // resolution + 1\n",
+    "    bar_list = []\n",
+    "\n",
+    "    for i in range(num_bars):\n",
+    "        bar_start_time = start_time + i * resolution\n",
+    "        bar_end_time = bar_start_time + resolution\n",
+    "        bar_ticks = ticks[(ticks[:, 0] >= bar_start_time) & (ticks[:, 0] < bar_end_time)]\n",
+    "        \n",
+    "        if bar_ticks.shape[0] == 0:\n",
+    "            continue  # Skip this bar as there are no ticks\n",
+    "\n",
+    "        # Calculate OHLCV values\n",
+    "        open_price = bar_ticks[0, 1]  # open\n",
+    "        high_price = np.max(bar_ticks[:, 1])  # high\n",
+    "        low_price = np.min(bar_ticks[:, 1])  # low\n",
+    "        close_price = bar_ticks[-1, 1]  # close\n",
+    "        volume = np.sum(bar_ticks[:, 2])  # volume\n",
+    "        bar_time = bar_start_time  # timestamp for the bar\n",
+    "\n",
+    "        bar_list.append([open_price, high_price, low_price, close_price, volume, bar_time])\n",
+    "\n",
+    "    # Convert list to numpy array\n",
+    "    if bar_list:\n",
+    "        ohlcv = np.array(bar_list)\n",
+    "    else:\n",
+    "        ohlcv = np.empty((0, 6))  # return an empty array if no bars were created\n",
+    "\n",
+    "    return ohlcv\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "DatetimeIndex: 190261 entries, 2024-04-22 13:30:00.267711+00:00 to 2024-04-22 19:59:59.987614+00:00\n",
+      "Data columns (total 6 columns):\n",
+      " #   Column      Non-Null Count   Dtype  \n",
+      "---  ------      --------------   -----  \n",
+      " 0   exchange    190261 non-null  object \n",
+      " 1   price       190261 non-null  float64\n",
+      " 2   size        190261 non-null  float64\n",
+      " 3   id          190261 non-null  int64  \n",
+      " 4   conditions  190261 non-null  object \n",
+      " 5   tape        190261 non-null  object \n",
+      "dtypes: float64(2), int64(1), object(3)\n",
+      "memory usage: 10.2+ MB\n"
+     ]
+    }
+   ],
+   "source": [
+    "df.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['C', 'O', '4', 'B', '7', 'V', 'P', 'W', 'U', 'Z', 'F']\n",
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "DatetimeIndex: 143751 entries, 2024-04-22 13:30:00.300501+00:00 to 2024-04-22 19:59:59.987614+00:00\n",
+      "Data columns (total 6 columns):\n",
+      " #   Column      Non-Null Count   Dtype  \n",
+      "---  ------      --------------   -----  \n",
+      " 0   exchange    143751 non-null  object \n",
+      " 1   price       143751 non-null  float64\n",
+      " 2   size        143751 non-null  float64\n",
+      " 3   id          143751 non-null  int64  \n",
+      " 4   conditions  143751 non-null  object \n",
+      " 5   tape        143751 non-null  object \n",
+      "dtypes: float64(2), int64(1), object(3)\n",
+      "memory usage: 7.7+ MB\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exchange</th>\n",
+       "      <th>price</th>\n",
+       "      <th>size</th>\n",
+       "      <th>id</th>\n",
+       "      <th>conditions</th>\n",
+       "      <th>tape</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>timestamp</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.300501+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241117014</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.305439+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241117496</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.314520+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241118034</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.335201+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241121369</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 13:30:00.346219+00:00</th>\n",
+       "      <td>D</td>\n",
+       "      <td>37.005</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>71675241122389</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.902614+00:00</th>\n",
+       "      <td>V</td>\n",
+       "      <td>37.750</td>\n",
+       "      <td>1100.0</td>\n",
+       "      <td>56480705310575</td>\n",
+       "      <td>[ ]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.977134+00:00</th>\n",
+       "      <td>N</td>\n",
+       "      <td>37.745</td>\n",
+       "      <td>300.0</td>\n",
+       "      <td>52983559963478</td>\n",
+       "      <td>[ ]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.977137+00:00</th>\n",
+       "      <td>N</td>\n",
+       "      <td>37.740</td>\n",
+       "      <td>7300.0</td>\n",
+       "      <td>52983559963696</td>\n",
+       "      <td>[ ]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.978626+00:00</th>\n",
+       "      <td>V</td>\n",
+       "      <td>37.750</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>56480706886228</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2024-04-22 19:59:59.987614+00:00</th>\n",
+       "      <td>N</td>\n",
+       "      <td>37.745</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>52983559963958</td>\n",
+       "      <td>[ , I]</td>\n",
+       "      <td>A</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>143751 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                 exchange   price    size              id  \\\n",
+       "timestamp                                                                   \n",
+       "2024-04-22 13:30:00.300501+00:00        D  37.005     1.0  71675241117014   \n",
+       "2024-04-22 13:30:00.305439+00:00        D  37.005     1.0  71675241117496   \n",
+       "2024-04-22 13:30:00.314520+00:00        D  37.005     1.0  71675241118034   \n",
+       "2024-04-22 13:30:00.335201+00:00        D  37.005     1.0  71675241121369   \n",
+       "2024-04-22 13:30:00.346219+00:00        D  37.005     1.0  71675241122389   \n",
+       "...                                   ...     ...     ...             ...   \n",
+       "2024-04-22 19:59:59.902614+00:00        V  37.750  1100.0  56480705310575   \n",
+       "2024-04-22 19:59:59.977134+00:00        N  37.745   300.0  52983559963478   \n",
+       "2024-04-22 19:59:59.977137+00:00        N  37.740  7300.0  52983559963696   \n",
+       "2024-04-22 19:59:59.978626+00:00        V  37.750    16.0  56480706886228   \n",
+       "2024-04-22 19:59:59.987614+00:00        N  37.745    30.0  52983559963958   \n",
+       "\n",
+       "                                 conditions tape  \n",
+       "timestamp                                         \n",
+       "2024-04-22 13:30:00.300501+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.305439+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.314520+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.335201+00:00     [ , I]    A  \n",
+       "2024-04-22 13:30:00.346219+00:00     [ , I]    A  \n",
+       "...                                     ...  ...  \n",
+       "2024-04-22 19:59:59.902614+00:00        [ ]    A  \n",
+       "2024-04-22 19:59:59.977134+00:00        [ ]    A  \n",
+       "2024-04-22 19:59:59.977137+00:00        [ ]    A  \n",
+       "2024-04-22 19:59:59.978626+00:00     [ , I]    A  \n",
+       "2024-04-22 19:59:59.987614+00:00     [ , I]    A  \n",
+       "\n",
+       "[143751 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "excludes = cfh.config_handler.get_val('AGG_EXCLUDED_TRADES')\n",
+    "print(excludes)\n",
+    "#excludes = [\"F\", \"I\"]\n",
+    "# FILTER EXCLUDED TRADES\n",
+    "# Filter rows to exclude those where 'conditions' contains 'F' or 'I'\n",
+    "# This simplifies the logic by directly using ~ (bitwise not operator) with np.isin\n",
+    "df = df[~df['conditions'].apply(lambda x: np.isin(x, excludes).any())]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/8p/dwqnp65s0s77jdbm4_6z4vp80000gn/T/ipykernel_52602/3341929382.py:2: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
+      "  structured_array = np.array(list(zip(df.index, df['price'], df['size'])),\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('2024-04-22T13:30:00.300501000', 37.005, 1.0e+00)\n",
+      " ('2024-04-22T13:30:00.305439000', 37.005, 1.0e+00)\n",
+      " ('2024-04-22T13:30:00.314520000', 37.005, 1.0e+00) ...\n",
+      " ('2024-04-22T19:59:59.977137000', 37.74 , 7.3e+03)\n",
+      " ('2024-04-22T19:59:59.978626000', 37.75 , 1.6e+01)\n",
+      " ('2024-04-22T19:59:59.987614000', 37.745, 3.0e+01)]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "array([('2024-04-22T13:30:00.300501000', 37.005, 1.0e+00),\n",
+       "       ('2024-04-22T13:30:00.305439000', 37.005, 1.0e+00),\n",
+       "       ('2024-04-22T13:30:00.314520000', 37.005, 1.0e+00), ...,\n",
+       "       ('2024-04-22T19:59:59.977137000', 37.74 , 7.3e+03),\n",
+       "       ('2024-04-22T19:59:59.978626000', 37.75 , 1.6e+01),\n",
+       "       ('2024-04-22T19:59:59.987614000', 37.745, 3.0e+01)],\n",
+       "      dtype=[('timestamp', '<M8[ns]'), ('price', '<f8'), ('size', '<f8')])"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Creating a structured array with the timestamp as the first element\n",
+    "structured_array = np.array(list(zip(df.index, df['price'], df['size'])),\n",
+    "                            dtype=[('timestamp', 'datetime64[ns]'), ('price', 'float'), ('size', 'float')])\n",
+    "\n",
+    "print(structured_array)\n",
+    "structured_array\n",
+    "\n",
+    "# ticks = df[['index', 'price', 'size']].to_numpy()\n",
+    "# # ticks[:, 0] = pd.to_datetime(ticks[:, 0]).astype('int64') // 1_000_000_000  # \n",
+    "# ticks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "resolution_seconds = 1  # 1 second resolution\n",
+    "ohlcv_data = ohlcv_bars(structured_array, resolution_seconds)\n",
+    "\n",
+    "# Converting the result back to DataFrame for better usability\n",
+    "ohlcv_df = pd.DataFrame(ohlcv_data, columns=['Open', 'High', 'Low', 'Close', 'Volume', 'Time'])\n",
+    "ohlcv_df['Time'] = pd.to_datetime(ohlcv_df['Time'], unit='s')  # Convert timestamps back to datetime\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

restart.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # file: restart.sh
-
+ 
 # Usage: ./restart.sh [test|prod|all]
 
 # Define server addresses

v2realbot/backtesting/backtester.py

@@ -524,7 +524,7 @@ class Backtester:
             if actual_minus_reserved <= 0:
                 cena = price if price else self.get_last_price(time, self.symbol)
                 if (self.cash - reserved_price - float(int(size)*float(cena))) < 0:
-                    printanyway("not enough cash for shorting. cash",self.cash,"reserved",reserved,"available",self.cash-reserved,"needed",float(int(size)*float(cena)))
+                    printanyway("ERROR: not enough cash for shorting. cash",self.cash,"reserved",reserved,"available",self.cash-reserved,"needed",float(int(size)*float(cena)))
                     return -1               
 
         #check for available cash
@@ -550,7 +550,7 @@ class Backtester:
             if actual_plus_reserved_qty >= 0:
                 cena = price if price else self.get_last_price(time, self.symbol)
                 if (self.cash - reserved_price - float(int(size)*float(cena))) < 0:
-                    printanyway("not enough cash to buy long. cash",self.cash,"reserved_qty",reserved_qty,"reserved_price",reserved_price, "available",self.cash-reserved_price,"needed",float(int(size)*float(cena)))
+                    printanyway("ERROR: not enough cash to buy long. cash",self.cash,"reserved_qty",reserved_qty,"reserved_price",reserved_price, "available",self.cash-reserved_price,"needed",float(int(size)*float(cena)))
                     return -1
 
         id = str(uuid4())

v2realbot/common/model.py

@@ -5,7 +5,7 @@ from rich import print
 from typing import Any, Optional, List, Union
 from datetime import datetime, date
 from pydantic import BaseModel, Field
-from v2realbot.enums.enums import Mode, Account, SchedulerStatus, Moddus
+from v2realbot.enums.enums import Mode, Account, SchedulerStatus, Moddus, Market
 from alpaca.data.enums import Exchange
 
 
@@ -159,6 +159,7 @@ class RunManagerRecord(BaseModel):
     mode: Mode
     note: Optional[str] = None
     ilog_save: bool = False
+    market: Optional[Market] = Market.US
     bt_from: Optional[datetime] = None
     bt_to: Optional[datetime] = None
     #weekdays filter

v2realbot/common/transform.py

@@ -5,9 +5,7 @@ import v2realbot.controller.services as cs
 
 #prevede dict radku zpatky na objekt vcetme retypizace
 def row_to_runmanager(row: dict) -> RunManagerRecord:
-
     is_running = cs.is_runner_running(row['runner_id']) if row['runner_id'] else False
-
     res = RunManagerRecord(
         moddus=row['moddus'],
         id=row['id'],
@@ -17,6 +15,7 @@ def row_to_runmanager(row: dict) -> RunManagerRecord:
         account=row['account'],
         note=row['note'],
         ilog_save=bool(row['ilog_save']),
+        market=row['market'] if row['market'] is not None else None,
         bt_from=datetime.fromisoformat(row['bt_from']) if row['bt_from'] else None,
         bt_to=datetime.fromisoformat(row['bt_to']) if row['bt_to'] else None,
         weekdays_filter=[int(x) for x in row['weekdays_filter'].split(',')] if row['weekdays_filter'] else [],

v2realbot/controller/run_manager.py

@@ -172,14 +172,14 @@ def add_run_manager_record(new_record: RunManagerRecord):
         # Construct a suitable INSERT query based on your RunManagerRecord fields
         insert_query = """
             INSERT INTO run_manager (moddus, id, strat_id, symbol,account, mode, note,ilog_save,
-                                    bt_from, bt_to, weekdays_filter, batch_id,
+                                    market, bt_from, bt_to, weekdays_filter, batch_id,
                                     start_time, stop_time, status, last_processed,
                                     history, valid_from, valid_to, testlist_id)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?) 
         """
         values = [
             new_record.moddus, str(new_record.id), str(new_record.strat_id), new_record.symbol, new_record.account, new_record.mode, new_record.note,
-            int(new_record.ilog_save),
+            int(new_record.ilog_save), new_record.market,
             new_record.bt_from.isoformat() if new_record.bt_from is not None else None,
             new_record.bt_to.isoformat() if new_record.bt_to is not None else None,
             ",".join(str(x) for x in new_record.weekdays_filter) if new_record.weekdays_filter else None,

v2realbot/enums/enums.py

@@ -103,4 +103,10 @@ class StartBarAlign(str, Enum):
         RANDOM = first bar starts when first trade occurs
     """ 
     ROUND = "round"
-    RANDOM = "random"
+    RANDOM = "random"
+
+class Market(str, Enum):
+    US = "US"
+    CRYPTO = "CRYPTO"
+
+     

v2realbot/loader/aggregator_vectorized.py

@@ -0,0 +1,122 @@
+import pandas as pd
+import numpy as np
+from numba import jit
+from alpaca.data.historical import StockHistoricalDataClient
+from v2realbot.config import ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, DATA_DIR
+from alpaca.data.requests import StockTradesRequest
+import time
+from datetime import datetime
+from v2realbot.utils.utils import parse_alpaca_timestamp, ltp, zoneNY, send_to_telegram, fetch_calendar_data
+import pyarrow
+
+""""
+WIP - for later use
+
+"""""
+
+def fetch_stock_trades(symbol, start, end, max_retries=5, backoff_factor=1):
+    """
+    Attempts to fetch stock trades with exponential backoff. Raises an exception if all retries fail.
+
+    :param symbol: The stock symbol to fetch trades for.
+    :param start: The start time for the trade data.
+    :param end: The end time for the trade data.
+    :param max_retries: Maximum number of retries.
+    :param backoff_factor: Factor to determine the next sleep time.
+    :return: TradesResponse object.
+    :raises: ConnectionError if all retries fail.
+    """
+    client =  StockHistoricalDataClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY)
+    stockTradeRequest = StockTradesRequest(symbol_or_symbols=symbol, start=start, end=end)
+    last_exception = None
+
+    for attempt in range(max_retries):
+        try:
+            tradesResponse = client.get_stock_trades(stockTradeRequest)
+            print("Remote Fetch DAY DATA Complete", start, end)
+            return tradesResponse
+        except Exception as e:
+            print(f"Attempt {attempt + 1} failed: {e}")
+            last_exception = e
+            time.sleep(backoff_factor * (2 ** attempt))
+
+    print("All attempts to fetch data failed.")
+    raise ConnectionError(f"Failed to fetch stock trades after {max_retries} retries. Last exception: {str(last_exception)} and {format_exc()}")
+
+
+@jit(nopython=True)
+def ohlcv_bars(ticks, start_time, end_time, resolution):
+    """
+    Generate OHLCV bars from tick data, skipping intervals without trading activity.
+    
+    Parameters:
+    - ticks: numpy array with columns [timestamp, price, size]
+    - start_time: the start timestamp for bars (Unix timestamp)
+    - end_time: the end timestamp for bars (Unix timestamp)
+    - resolution: time resolution in seconds
+    
+    Returns:
+    - OHLCV bars as a numpy array
+    """
+    num_bars = (end_time - start_time) // resolution + 1
+    bar_list = []
+
+    for i in range(num_bars):
+        bar_start_time = start_time + i * resolution
+        bar_end_time = bar_start_time + resolution
+        bar_ticks = ticks[(ticks[:, 0] >= bar_start_time) & (ticks[:, 0] < bar_end_time)]
+        
+        if bar_ticks.shape[0] == 0:
+            continue  # Skip this bar as there are no ticks
+
+        # Calculate OHLCV values
+        open_price = bar_ticks[0, 1]  # open
+        high_price = np.max(bar_ticks[:, 1])  # high
+        low_price = np.min(bar_ticks[:, 1])  # low
+        close_price = bar_ticks[-1, 1]  # close
+        volume = np.sum(bar_ticks[:, 2])  # volume
+        bar_time = bar_start_time  # timestamp for the bar
+
+        bar_list.append([open_price, high_price, low_price, close_price, volume, bar_time])
+
+    # Convert list to numpy array
+    if bar_list:
+        ohlcv = np.array(bar_list)
+    else:
+        ohlcv = np.empty((0, 6))  # return an empty array if no bars were created
+
+    return ohlcv
+
+# Example usage
+if __name__ == '__main__':
+    # symbol = ["BAC"]
+    # #datetime in zoneNY 
+    # day_start = datetime(2024, 4, 22, 9, 30, 0)
+    # day_stop = datetime(2024, 4, 22, 16, 00, 0)
+
+    # day_start = zoneNY.localize(day_start)
+    # day_stop = zoneNY.localize(day_stop)
+
+    # tradesResponse = fetch_stock_trades(symbol, day_start, day_stop)
+
+    # df = tradesResponse.df
+    # df.to_parquet('trades_bac.parquet', engine='pyarrow')
+
+    df=pd.read_parquet('trades_bac.parquet',engine='pyarrow')
+    print(df)
+
+    #df = pd.read_csv('tick_data.csv')  # DF with tick data
+# Assuming 'df' is your DataFrame with columns 'time', 'price', 'size', 'condition'
+    exclude_conditions = ['ConditionA', 'ConditionB']  # Conditions to exclude
+    df_filtered = df[~df['condition'].isin(exclude_conditions)]
+    # Define your start and end times based on your trading session, ensure these are Unix timestamps
+    start_time = pd.to_datetime('2023-01-01 09:30:00').timestamp()
+    end_time = pd.to_datetime('2023-01-01 16:00:00').timestamp()
+    ticks = df[['time', 'price', 'size']].to_numpy()
+    ticks[:, 0] = pd.to_datetime(ticks[:, 0]).astype('int64') // 1_000_000_000  # Convert to Unix timestamp
+    resolution_seconds = 1  # 1 second resolution
+    ohlcv_data = ohlcv_bars(ticks, start_time, end_time, resolution_seconds)
+
+    # Converting the result back to DataFrame for better usability
+    ohlcv_df = pd.DataFrame(ohlcv_data, columns=['Open', 'High', 'Low', 'Close', 'Volume', 'Time'])
+    ohlcv_df['Time'] = pd.to_datetime(ohlcv_df['Time'], unit='s')  # Convert timestamps back to datetime

v2realbot/scheduler/ap_scheduler.py

@@ -2,7 +2,7 @@ from uuid import UUID
 from typing import Any, List, Tuple
 from uuid import UUID, uuid4
 from v2realbot.enums.enums import Moddus, SchedulerStatus, RecordType, StartBarAlign, Mode, Account, OrderSide
-from v2realbot.common.model import RunManagerRecord, StrategyInstance, RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest
+from v2realbot.common.model import RunManagerRecord, StrategyInstance, RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest, Market
 from v2realbot.utils.utils import validate_and_format_time, AttributeDict, zoneNY, zonePRG, safe_get, dict_replace_value, Store, parse_toml_string, json_serial, is_open_hours, send_to_telegram, concatenate_weekdays, transform_data
 from v2realbot.common.PrescribedTradeModel import Trade, TradeDirection, TradeStatus, TradeStoplossType
 from datetime import datetime
@@ -116,7 +116,8 @@ def initialize_jobs(run_manager_records: RunManagerRecord = None):
             scheduler.add_job(start_runman_record, start_trigger, id=f"scheduler_start_{record.id}", args=[record.id])
             scheduler.add_job(stop_runman_record, stop_trigger, id=f"scheduler_stop_{record.id}", args=[record.id])
                         
-    #scheduler.add_job(print_hello, 'interval', seconds=10, id=f"scheduler_testinterval")
+    #scheduler.add_job(print_hello, 'interval', seconds=10, id=
+    # f"scheduler_testinterval")
     scheduled_jobs = scheduler.get_jobs()
     print(f"APS jobs refreshed ({len(scheduled_jobs)})")
     current_jobs_dict = format_apscheduler_jobs(scheduled_jobs)
@@ -124,9 +125,9 @@ def initialize_jobs(run_manager_records: RunManagerRecord = None):
     return 0, current_jobs_dict
 
 #zastresovaci funkce resici error handling a printing
-def start_runman_record(id: UUID, market = "US", debug_date = None):
+def start_runman_record(id: UUID, debug_date = None):
     record = None
-    res, record, msg = _start_runman_record(id=id, market=market, debug_date=debug_date)
+    res, record, msg = _start_runman_record(id=id, debug_date=debug_date)
 
     if record is not None:
         market_time_now = datetime.now().astimezone(zoneNY) if debug_date is None else debug_date
@@ -165,8 +166,8 @@ def update_runman_record(record: RunManagerRecord):
         err_msg= f"STOP: Error updating {record.id} errir {set} with values {record}"
         return -2, err_msg#toto stopne zpracovani dalsich zaznamu pri chybe, zvazit continue
 
-def stop_runman_record(id: UUID, market = "US", debug_date = None):
-    res, record, msg = _stop_runman_record(id=id, market=market, debug_date=debug_date)
+def stop_runman_record(id: UUID, debug_date = None):
+    res, record, msg = _stop_runman_record(id=id, debug_date=debug_date)
     #results : 0 - ok, -1 not running/already running/not specific, -2 error
 
     #report vzdy zapiseme do history, pokud je record not None, pripadna chyba se stala po dotazeni recordu
@@ -196,7 +197,7 @@ def stop_runman_record(id: UUID, market = "US", debug_date = None):
         print(f"STOP JOB: {id} FINISHED")
 
 #start function that is called from the job
-def _start_runman_record(id: UUID, market = "US", debug_date = None):
+def _start_runman_record(id: UUID, debug_date = None):
     print(f"Start scheduled record {id}")
 
     record : RunManagerRecord = None
@@ -207,15 +208,16 @@ def _start_runman_record(id: UUID, market = "US", debug_date = None):
     
     record = result
 
-    if market is not None and market == "US":
-        res, sada = sch.get_todays_market_times(market=market, debug_date=debug_date)
+    if record.market == Market.US or record.market == Market.CRYPTO:
+        res, sada = sch.get_todays_market_times(market=record.market, debug_date=debug_date)
         if res == 0:
             market_time_now, market_open_datetime, market_close_datetime = sada
             print(f"OPEN:{market_open_datetime} CLOSE:{market_close_datetime}")
         else:
-            sada = f"Market {market} Error getting market times (CLOSED): " + str(sada)
+            sada = f"Market {record.market} Error getting market times (CLOSED): " + str(sada)
             return res, record, sada
-    
+    else:
+        print("Market type is unknown.")
     if cs.is_stratin_running(record.strat_id):
         return -1, record, f"Stratin {record.strat_id} is already running"
 
@@ -229,7 +231,7 @@ def _start_runman_record(id: UUID, market = "US", debug_date = None):
     return 0, record, record.runner_id
 
 #stop function that is called from the job
-def _stop_runman_record(id: UUID, market = "US", debug_date = None):
+def _stop_runman_record(id: UUID, debug_date = None):
     record = None
     #get all records
     print(f"Stopping record {id}")
@@ -304,5 +306,5 @@ if __name__ == "__main__":
     # print(f"CALL FINISHED, with {debug_date} RESULT: {res}, {result}")
 
 
-    res, result = stop_runman_record(id=id, market = "US", debug_date = debug_date)
+    res, result = stop_runman_record(id=id, debug_date = debug_date)
     print(f"CALL FINISHED, with {debug_date} RESULT: {res}, {result}")

v2realbot/scheduler/scheduler.py

@@ -2,10 +2,10 @@ import json
 import datetime
 import v2realbot.controller.services as cs
 import v2realbot.controller.run_manager as rm
-from v2realbot.common.model import RunnerView, RunManagerRecord, StrategyInstance, Runner, RunRequest, Trade, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, Bar, RunArchiveChange, TestList, ConfigItem, InstantIndicator, DataTablesRequest, AnalyzerInputs
+from v2realbot.common.model import RunnerView, RunManagerRecord, StrategyInstance, Runner, RunRequest, Trade, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, Bar, RunArchiveChange, TestList, ConfigItem, InstantIndicator, DataTablesRequest, AnalyzerInputs, Market
 from uuid import uuid4, UUID
-from v2realbot.utils.utils import json_serial, send_to_telegram, zoneNY, zonePRG, fetch_calendar_data
-from datetime import datetime, timedelta
+from v2realbot.utils.utils import json_serial, send_to_telegram, zoneNY, zonePRG, zoneUTC, fetch_calendar_data
+from datetime import datetime, timedelta, time
 from traceback import format_exc
 from rich import print
 import requests
@@ -18,9 +18,18 @@ from v2realbot.config import WEB_API_KEY
 #naplanovany jako samostatni job a triggerován pouze jednou v daný čas pro start a stop 
 #novy kod v aps_scheduler.py
 
-def get_todays_market_times(market = "US", debug_date = None):
+def is_US_market_day(date):
+    cal_dates = fetch_calendar_data(date, date)
+    if len(cal_dates) == 0:
+        print("Today is not a market day.")
+        return False, cal_dates
+    else:
+        print("Market is open")
+        return True, cal_dates
+
+def get_todays_market_times(market, debug_date = None):
     try:
-        if market == "US":
+        if market == Market.US:
             #zjistit vsechny podminky - mozna loopovat - podminky jsou vlevo
             if debug_date is not None:
                 nowNY = debug_date
@@ -28,17 +37,20 @@ def get_todays_market_times(market = "US", debug_date = None):
                 nowNY = datetime.now().astimezone(zoneNY)
             nowNY_date = nowNY.date()
             #is market open - nyni pouze US
-            cal_dates = fetch_calendar_data(nowNY_date, nowNY_date)
-
-            if len(cal_dates) == 0:
-                print("No Market Day today")
-                return -1, "Market Closed"
+            stat, calendar_dates = is_US_market_day(nowNY_date)
+            if stat:
             #zatim podpora pouze main session
-
             #pouze main session
-            market_open_datetime = zoneNY.localize(cal_dates[0].open)
-            market_close_datetime = zoneNY.localize(cal_dates[0].close)
-            return 0, (nowNY, market_open_datetime, market_close_datetime)
+                market_open_datetime = zoneNY.localize(calendar_dates[0].open)
+                market_close_datetime = zoneNY.localize(calendar_dates[0].close)
+                return 0, (nowNY, market_open_datetime, market_close_datetime)
+            else:
+                return -1, "Market is closed."
+        elif market == Market.CRYPTO:
+            now_market_datetime = datetime.now().astimezone(zoneUTC)
+            market_open_datetime = datetime.combine(datetime.now(), time.min)
+            matket_close_datetime = datetime.combine(datetime.now(), time.max)
+            return 0, (now_market_datetime, market_open_datetime, matket_close_datetime)
         else:
             return -1, "Market not supported"
     except Exception as e:

v2realbot/static/index.html

@@ -347,6 +347,7 @@
                                 <th>testlist_id</th>
                                 <th>Running</th>
                                 <th>RunnerId</th>
+                                <th>Market</th>
                             </tr>
                         </thead>
                         <tbody></tbody>

v2realbot/static/js/tables/archivetable/functions.js

@@ -495,6 +495,12 @@ function refresh_logfile() {
                     readOnly: true
                 });
                 });
+                // Focus at the end of the file:
+                const model = editorLog.getModel();
+                const lastLineNumber = model.getLineCount();
+                const lastLineColumn = model.getLineMaxColumn(lastLineNumber);
+                editorLog.setPosition({ lineNumber: lastLineNumber, column: lastLineColumn });
+                editorLog.revealPosition({ lineNumber: lastLineNumber, column: lastLineColumn });
         },
         error: function(xhr, status, error) {
             var err = eval("(" + xhr.responseText + ")");

v2realbot/static/js/tables/runmanager/init.js

@@ -45,7 +45,8 @@ function initialize_runmanagerRecords() {
                         {data: 'valid_to', visible: true},
                         {data: 'testlist_id', visible: true},
                         {data: 'strat_running', visible: true},
-                        {data: 'runner_id', visible: true},                  
+                        {data: 'runner_id', visible: true},
+                        {data: 'market', visible: true},                  
                     ],
             paging: true,
             processing: true,