{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Test persistance\n", "\n", "ohlcv and trades persistence with bar type and trade filtering and minsize support\n", "\n", "```\n", "/OHLCV/\n", " ├── {bar_type}/ (1s)\n", " │ ├── {resolution}/\n", " │ │ ├── {filtered_trades}-{min_trade_size}/\n", " │ │ │ ├── {day}/\n", " │ │ │ │ └── hashedname.parquet\n", "```" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "
\n", "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "
\n", "
\n", "This is the init_notebook_mode cell from ITables v2.0.1
\n", "(you should not see this message - is your notebook trusted?)\n", "
\n", "
\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "trades_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n", "trades_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n", "trades_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n", "trades_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n", "trades_df-BAC-2024-05-15T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n", "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n", "ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n" ] } ], "source": [ "from v2realbot.tools.loadbatch import load_batch\n", "from v2realbot.utils.utils import zoneNY\n", "import pandas as pd\n", "import numpy as np\n", "import vectorbtpro as vbt\n", "from itables import init_notebook_mode, show\n", "import datetime\n", "from itertools import product\n", "from v2realbot.config import ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, DATA_DIR\n", "import v2realbot.utils.config_handler as cfh\n", "init_notebook_mode(all_interactive=True)\n", "from v2realbot.enums.enums import BarType\n", "\n", "vbt.settings.set_theme(\"dark\")\n", "vbt.settings['plotting']['layout']['width'] = 1280\n", "vbt.settings.plotting.auto_rangebreaks = True\n", "# Set the option to display with pagination\n", "pd.set_option('display.notebook_repr_html', True)\n", "pd.set_option('display.max_rows', 10) # Number of rows per page\n", "\n", "# Define the market open and close times\n", "market_open = datetime.time(9, 30)\n", "market_close = datetime.time(16, 0)\n", "entry_window_opens = 1\n", "entry_window_closes = 370\n", "\n", "forced_exit_start = 380\n", "forced_exit_end = 390\n", "\n", "#LOAD FROM BATCH\n", "# res, df = load_batch(batch_id=\"f1ac6651\", #138170bc 0fb5043a bde6d0be f1ac6651\n", "# space_resolution_evenly=False,\n", "# indicators_columns=[\"Rsi14\"],\n", "# main_session_only=True,\n", "# verbose = False)\n", "# if res < 0:\n", "# print(\"Error\" + str(res) + str(df))\n", "# df = df[\"bars\"]\n", "\n", "# basic_data = vbt.Data.from_data(vbt.symbol_dict({\"BAC\": df}), tz_convert=zoneNY)\n", "# #m1_data = basic_data[['Open', 'High', 'Low', 'Close', 'Volume']]\n", "# basic_data = basic_data.transform(lambda df: df.between_time('09:30', '16:00'))\n", "# #basic_data.info()\n", "\n", "#LOAD FROM PARQUET\n", "#list all files is dir directory with parquet extension\n", "dir = DATA_DIR + \"/notebooks/\"\n", "import os\n", "files = [f for f in os.listdir(dir) if f.endswith(\".parquet\")]\n", "print('\\n'.join(map(str, files)))\n", "file_name = \"ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\"\n", "ohlcv_df = pd.read_parquet(dir+file_name,engine='pyarrow')\n", "\n", "#filter ohlcv_df to certain date range (assuming datetime index)\n", "#ohlcv_df = ohlcv_df.loc[\"2024-05-14 09:30\":\"2024-05-15 09:35\"]\n", "\n", "#add vwap column to ohlcv_df\n", "#ohlcv_df[\"hlcc4\"] = (ohlcv_df[\"close\"] + ohlcv_df[\"high\"] + ohlcv_df[\"low\"] + ohlcv_df[\"close\"]) / 4\n", "\n", "basic_data = vbt.Data.from_data(vbt.symbol_dict({\"BAC\": ohlcv_df}), tz_convert=zoneNY)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "\n", "\n", "
close
time
\n", "\n", "
\n", "Loading ITables v2.0.1 from the init_notebook_mode cell...\n", "(need help?)
\n", "\n" ], "text/plain": [ "time\n", "2023-01-31 00:00:00-05:00 34.365180\n", "2023-02-28 00:00:00-05:00 35.438030\n", "2023-03-31 00:00:00-04:00 29.484466\n", "2023-04-30 00:00:00-04:00 29.031967\n", "2023-05-31 00:00:00-04:00 27.928008\n", " ... \n", "2024-01-31 00:00:00-05:00 33.221474\n", "2024-02-29 00:00:00-05:00 33.582925\n", "2024-03-31 00:00:00-04:00 36.142510\n", "2024-04-30 00:00:00-04:00 36.945401\n", "2024-05-31 00:00:00-04:00 38.438940\n", "Freq: ME, Name: close, Length: 17, dtype: float64" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#basic_data.data[\"BAC\"].info()\n", "#ohlcv_df group by week number of rows\n", "# ohlcv_df['close'].groupby(pd.Grouper(freq='ME')).mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#trade filtering\n", "exclude_conditions = cfh.config_handler.get_val('AGG_EXCLUDED_TRADES') #standard ['C','O','4','B','7','V','P','W','U','Z','F']\n", "minsize = 100\n", "exclude_conditions_str = ''.join(exclude_conditions)\n", "exclude_conditions_str" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "basic_data.data[\"BAC\"].info()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Aim is to store\n", "OHLCV grouped by symbol, day, resolution\n", "and \n", "bar type\n", "excluded_conditions\n", "minsize\n", "main session" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "bartype= BarType.TIME\n", "resolution = \"1s\"\n", "trade_filter = exclude_conditions_str+\"-\"+str(minsize)\n", "dir = \"/OHLCV/\"+bartype+\"/\"+resolution+\"/\"+trade_filter+\"/\"\n", "#dir = DATA_DIR + dir\n", "basic_data.to_parquet(partition_by=\"day\", keep_groupby_names=False, path_or_buf=dir, mkdir_kwargs=dict(mkdir=True)) \n", "#partition_by=\"day\",\n", "\n", "#naloaduje partitionvana 1s data skrz 90 dni za 2s\n", "#day_data = vbt.ParquetData.pull(\"BAC\", paths=dir, filters=[(\"group\", \">\", \"2024-01-02\"),(\"group\", \"<=\", \"2024-01-09\")]) #, \n", "# day_data[\"2024-05-01\":\"2024-05-14\"].get()\n", "\n", "# day_data.data[\"BAC\"].info()\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#naloaduje partitionvana 1s data skrz 90 dni za 2s\n", "day_data = vbt.ParquetData.pull(\"BAC\", paths=dir, filters=[(\"group\", \">=\", \"2024-01-02\"),(\"group\", \"<=\", \"2024-01-09\")]) #, \n", "# day_data[\"2024-05-01\":\"2024-05-14\"].get()\n", "\n", "day_data.data[\"BAC\"].info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "close = basic_data.close\n", "#group by close by day, using pandas grouper\n", "#close.groupby(pd.Grouper(freq='ME')).mean()\n", "\n", "#using Grouper of vectorbtpro\n", "#close.vbt.group_by(pd.Grouper(freq='ME')).mean()\n", "\n", "#basic_data.wrapper.get_columns()\n", "basic_data.wrapper.get_freq()\n", "# vbt.pdir(basic_data.wrapper)\n", "# basic_data.wrapper\n", "basic_data.wrapper.grouper.is_grouped()\n", "\n", "vbt.pdir(basic_data.wrapper.grouper)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "grouper = basic_data.wrapper.index.vbt.get_grouper(\"ME\")\n", "\n", "for group, group_idx in grouper:\n", " print(group, group_idx)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "df" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#prevede 1milion dat (6mes 1s) na dict za 10ss\n", "df = day_data.data[\"BAC\"]\n", "df_dict = df.to_dict(orient='list')\n", "\n", "# Convert the index (which is the time) to a list of float timestamps\n", "df_dict['time'] = [timestamp.timestamp() for timestamp in df.index]\n", "\n", "df_dict" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.11" } }, "nbformat": 4, "nbformat_minor": 2 }