{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# How to load and save data to remote database\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "None\n", "Loaded env variables from file None\n" ] }, { "data": { "text/html": [ "
Activating profile profile1\n",
       "
\n" ], "text/plain": [ "Activating profile profile1\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from lightweight_charts import Panel, chart, PlotDFAccessor\n", "from v2realbot.utils.utils import zoneNY\n", "import pandas as pd\n", "import numpy as np\n", "import vectorbtpro as vbt\n", "# from itables import init_notebook_mode, show\n", "import datetime\n", "from itertools import product\n", "from v2realbot.config import DATA_DIR\n", "from IPython.display import display\n", "\n", "# init_notebook_mode(all_interactive=True)\n", "\n", "vbt.settings.set_theme(\"dark\")\n", "vbt.settings['plotting']['layout']['width'] = 1280\n", "vbt.settings.plotting.auto_rangebreaks = True\n", "# Set the option to display with pagination\n", "pd.set_option('display.notebook_repr_html', True)\n", "pd.set_option('display.max_rows', 10) # Number of rows per page" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetch STOCKS from db" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ac5230e6571e4664ad606c118fa0e9f5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "100%|##########| 1/1 [00:10<00:00, 10.52s/it, symbol=BAC]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "DatetimeIndex: 288034 entries, 2024-08-01 09:30:00-04:00 to 2024-09-04 15:59:59-04:00\n", "Data columns (total 10 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 open 288034 non-null float64 \n", " 1 high 288034 non-null float64 \n", " 2 low 288034 non-null float64 \n", " 3 close 288034 non-null float64 \n", " 4 volume 288034 non-null float64 \n", " 5 trades 288034 non-null float64 \n", " 6 updated 288034 non-null datetime64[ns, UTC]\n", " 7 vwap 288034 non-null float64 \n", " 8 buyvolume 288034 non-null float64 \n", " 9 sellvolume 288034 non-null float64 \n", "dtypes: datetime64[ns, UTC](1), float64(9)\n", "memory usage: 24.2 MB\n" ] } ], "source": [ "#fetching US-STOCKS ohlcv_1s\n", "from lib.db import Connection\n", "SYMBOL = \"BAC\"\n", "SCHEMA = \"ohlcv_1s\" #time based 1s other options ohlcv_vol_200 (volume based ohlcv with resolution of 200), ohlcv_renko_20 (renko with 20 bricks size) ...\n", "DB = \"market_data\"\n", "\n", "con = Connection(db_name=DB, default_schema=SCHEMA, create_db=True)\n", "data = con.pull(symbols=[SYMBOL], schema=SCHEMA,start=\"2024-08-01\", end=\"2024-09-05\", tz_convert='America/New_York')\n", "\n", "data.data[SYMBOL].info()\n", "#1month 1s data - 15s - 24MB" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "data.xloc[\"2024-08-05\":\"2024-08-10\"].data[SYMBOL].close.lw.plot()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetch CRYPTO" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "55ebcdb7940b4645a61fe3aa2e2604e4", "version_major": 2, "version_minor": 0 }, "text/plain": [ "100%|##########| 1/1 [00:10<00:00, 10.14s/it, symbol=BTC/USD]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "#fetching CRYPTO from db from krakenfutures exchange\n", "from lib.db import Connection\n", "SYMBOL = \"BTC/USD\"\n", "con = Connection(db_name=\"market_data\", default_schema=\"krakenfutures\", create_db=True)\n", "data = con.pull(symbols=[SYMBOL], start=\"2024-01-01\", end=\"2025-01-01\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Load data from local CACHE" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "trades_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n", "trades_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n", "trades_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n", "trades_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n", "trades_df-BAC-2024-10-03T09:30:00-2024-10-16T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n", "trades_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n", "trades_df-BAC-2024-05-15T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-BAC-2024-10-03T09:30:00-2024-10-16T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n", "ohlcv_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n", "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n", "ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n", "ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n", "ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T15_30_00-47BCFOPUVWZ-100.parquet\n", "\n", "DatetimeIndex: 114097 entries, 2024-10-03 09:30:00-04:00 to 2024-10-16 15:59:59-04:00\n", "Data columns (total 10 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 open 114097 non-null float64 \n", " 1 high 114097 non-null float64 \n", " 2 low 114097 non-null float64 \n", " 3 close 114097 non-null float64 \n", " 4 volume 114097 non-null float64 \n", " 5 trades 114097 non-null float64 \n", " 6 updated 114097 non-null datetime64[ns, US/Eastern]\n", " 7 vwap 114097 non-null float64 \n", " 8 buyvolume 114097 non-null float64 \n", " 9 sellvolume 114097 non-null float64 \n", "dtypes: datetime64[ns, US/Eastern](1), float64(9)\n", "memory usage: 9.6 MB\n" ] } ], "source": [ "# Define the market open and close times\n", "market_open = datetime.time(9, 30)\n", "market_close = datetime.time(16, 0)\n", "entry_window_opens = 1\n", "entry_window_closes = 370\n", "forced_exit_start = 380\n", "forced_exit_end = 390\n", "\n", "#LOAD FROM local PARQUET cache\n", "#list all files is dir directory with parquet extension\n", "dir = DATA_DIR + \"/notebooks/\"\n", "import os\n", "files = [f for f in os.listdir(dir) if f.endswith(\".parquet\")]\n", "print('\\n'.join(map(str, files)))\n", "#file_name = \"ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T15_30_00-47BCFOPUVWZ-100.parquet\"\n", "#file_name = \"ohlcv_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\"\n", "file_name = \"ohlcv_df-BAC-2024-10-03T09:30:00-2024-10-16T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\"\n", "ohlcv_df = pd.read_parquet(dir+file_name,engine='pyarrow')\n", "#filter ohlcv_df to certain date range (assuming datetime index)\n", "#ohlcv_df = ohlcv_df.loc[\"2024-02-12 9:30\":\"2024-02-16 16:00\"]\n", "\n", "#add vwap column to ohlcv_df\n", "#ohlcv_df[\"hlcc4\"] = (ohlcv_df[\"close\"] + ohlcv_df[\"high\"] + ohlcv_df[\"low\"] + ohlcv_df[\"close\"]) / 4\n", "\n", "basic_data = vbt.Data.from_data(vbt.symbol_dict({\"BAC\": ohlcv_df}), tz_convert=zoneNY)\n", "#ohlcv_df= None\n", "# basic_data.wrapper.index.normalize().nunique()\n", "# bd = basic_data.transform(lambda df: df[:1000000])\n", "basic_data.data[\"BAC\"].info()\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "basic_data.wrapper.index.normalize().nunique()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Saving to DATABASE" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "from lib.db import Connection\n", "SYMBOL = \"BAC\"\n", "SCHEMA = \"ohlcv_1s\" #time based 1s other options ohlcv_vol_200 (volume based ohlcv with resolution of 200), ohlcv_renko_20 (renko with 20 bricks size) ...\n", "DB = \"market_data\"\n", "\n", "try:\n", " con = Connection(db_name=DB, default_schema=SCHEMA, create_db=False)\n", " #con.force_rollback_all() # Force cleanup before attempting to save\n", " con.save(basic_data, chunksize=500)\n", "except Exception as e:\n", " print(f\"Error occurred: {str(e)}\")\n", "\n", "#63sec to insert remotely 5mio records\n" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "DatetimeIndex: 4549772 entries, 2023-01-03 09:30:01-05:00 to 2024-05-24 15:59:59-04:00\n", "Data columns (total 10 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 open float64 \n", " 1 high float64 \n", " 2 low float64 \n", " 3 close float64 \n", " 4 volume float64 \n", " 5 trades float64 \n", " 6 updated datetime64[ns, US/Eastern]\n", " 7 vwap float64 \n", " 8 buyvolume float64 \n", " 9 sellvolume float64 \n", "dtypes: datetime64[ns, US/Eastern](1), float64(9)\n", "memory usage: 381.8 MB\n" ] } ], "source": [ "basic_data.data[\"BAC\"].info()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Add resample function to custom columns" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Resample to different resolution" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from vectorbtpro.utils.config import merge_dicts, Config, HybridConfig\n", "from vectorbtpro import _typing as tp\n", "from vectorbtpro.generic import nb as generic_nb\n", "\n", "_feature_config: tp.ClassVar[Config] = HybridConfig(\n", " {\n", " \"buyvolume\": dict(\n", " resample_func=lambda self, obj, resampler: obj.vbt.resample_apply(\n", " resampler,\n", " generic_nb.sum_reduce_nb,\n", " )\n", " ),\n", " \"sellvolume\": dict(\n", " resample_func=lambda self, obj, resampler: obj.vbt.resample_apply(\n", " resampler,\n", " generic_nb.sum_reduce_nb,\n", " )\n", " ),\n", " \"trades\": dict(\n", " resample_func=lambda self, obj, resampler: obj.vbt.resample_apply(\n", " resampler,\n", " generic_nb.sum_reduce_nb,\n", " )\n", " )\n", " }\n", ")\n", "\n", "basic_data._feature_config = _feature_config" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "s1data = basic_data[['open', 'high', 'low', 'close', 'volume','vwap','buyvolume','trades','sellvolume']]\n", "\n", "s5data = s1data.resample(\"12s\")\n", "s5data = s5data.transform(lambda df: df.between_time('09:30', '16:00').dropna())\n", "\n", "t1data = basic_data[['open', 'high', 'low', 'close', 'volume','vwap','buyvolume','trades','sellvolume']].resample(\"1T\")\n", "t1data = t1data.transform(lambda df: df.between_time('09:30', '16:00').dropna())\n", "# t1data.data[\"BAC\"].info()\n", "\n", "t30data = basic_data[['open', 'high', 'low', 'close', 'volume','vwap','buyvolume','trades','sellvolume']].resample(\"30T\")\n", "t30data = t30data.transform(lambda df: df.between_time('09:30', '16:00').dropna())\n", "# t30data.data[\"BAC\"].info()\n", "\n", "s1close = s1data.close\n", "t1close = t1data.close\n", "t30close = t30data.close\n", "t30volume = t30data.volume" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from lightweight_charts import JupyterChart, chart, Panel, PlotAccessor\n", "s5data.close.lw.plot()\n", "\n", "# pane1 = Panel(\n", "# ohlcv=(s5data.ohlcv.get(),))\n", "\n", "# # Create the chart with the panel\n", "# ch = chart([pane1], title=\"Chart\", sync=True, session=None, size=\"s\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
openhighlowclosevolumevwapbuyvolumetradessellvolume
time
2024-02-12 09:30:00-05:0033.00033.0033.0033.000800.033.0000000.02.00.0
2024-02-12 09:30:01-05:0033.02033.0533.0033.010267365.033.020039265765.025.01000.0
2024-02-12 09:30:02-05:0033.00033.0233.0033.0151100.033.009545500.09.0600.0
2024-02-12 09:30:03-05:0033.00533.0832.9933.0807508.033.0278981970.032.03638.0
2024-02-12 09:30:05-05:0033.06033.0633.0633.060500.033.0600000.05.0100.0
\n", "
" ], "text/plain": [ " open high low close volume vwap \\\n", "time \n", "2024-02-12 09:30:00-05:00 33.000 33.00 33.00 33.000 800.0 33.000000 \n", "2024-02-12 09:30:01-05:00 33.020 33.05 33.00 33.010 267365.0 33.020039 \n", "2024-02-12 09:30:02-05:00 33.000 33.02 33.00 33.015 1100.0 33.009545 \n", "2024-02-12 09:30:03-05:00 33.005 33.08 32.99 33.080 7508.0 33.027898 \n", "2024-02-12 09:30:05-05:00 33.060 33.06 33.06 33.060 500.0 33.060000 \n", "\n", " buyvolume trades sellvolume \n", "time \n", "2024-02-12 09:30:00-05:00 0.0 2.0 0.0 \n", "2024-02-12 09:30:01-05:00 265765.0 25.0 1000.0 \n", "2024-02-12 09:30:02-05:00 500.0 9.0 600.0 \n", "2024-02-12 09:30:03-05:00 1970.0 32.0 3638.0 \n", "2024-02-12 09:30:05-05:00 0.0 5.0 100.0 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s1data.data[\"BAC\"].head()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "#resample on specific index \n", "resampler = vbt.Resampler(t30data.index, s1data.index, source_freq=\"30T\", target_freq=\"1s\")\n", "t30close_realigned = t30close.vbt.realign_closing(resampler)\n", "\n", "#resample 1min to s\n", "resampler_s = vbt.Resampler(t1data.index, s1data.index, source_freq=\"1T\", target_freq=\"1s\")\n", "t1close_realigned = t1close.vbt.realign_closing(resampler_s)" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.11" } }, "nbformat": 4, "nbformat_minor": 2 }