{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Load data\n", "\n", "Make sure you have .env file in ttools or any parent dir with your Alpaca keys.\n", "\n", "```bash\n", "ACCOUNT1_LIVE_API_KEY=api_key\n", "ACCOUNT1_LIVE_SECRET_KEY=secret_key\n", "```\n", "\n", "### Cache directories\n", "\n", "Daily trade files - DATADIR/tradecache\n", "Agg data cache - DATADIR/aggcache\n", "\n", "`DATADIR` - user_data_dir from appdirs library - see `config.py`" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TTOOLS: Loaded env variables from file /Users/davidbrazda/Documents/Development/python/.env\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "from ttools.utils import AggType\n", "from datetime import datetime\n", "from ttools.aggregator_vectorized import generate_time_bars_nb, aggregate_trades\n", "from ttools.loaders import load_data, prepare_trade_cache, fetch_daily_stock_trades\n", "from ttools.utils import zoneNY\n", "import vectorbtpro as vbt\n", "from lightweight_charts import PlotDFAccessor, PlotSRAccessor\n", "\n", "\n", "vbt.settings.set_theme(\"dark\")\n", "vbt.settings['plotting']['layout']['width'] = 1280\n", "vbt.settings.plotting.auto_rangebreaks = True\n", "# Set the option to display with pagination\n", "pd.set_option('display.notebook_repr_html', True)\n", "pd.set_option('display.max_rows', 10) # Number of rows per page\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetching aggregated data\n", "\n", "Available aggregation types: \n", "- time based bars - AggType.OHLCV\n", "- volume based bars - AggType.OHLCV_VOL, resolution = volume threshold\n", "- dollar based bars - AggType.OHLCV_DOL, resolution = dollar threshold\n", "- renko bars - AggType.OHLCV_RENKO resolution = bricksize" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
openhighlowclosevolume
time
2024-09-16 04:01:24-04:00562.22562.22562.22562.22200.0
2024-09-16 04:02:24-04:00562.17562.17562.17562.17293.0
2024-09-16 04:04:36-04:00562.54562.54562.54562.54100.0
2024-09-16 04:10:00-04:00562.39562.39562.39562.39102.0
2024-09-16 04:10:24-04:00562.44562.44562.44562.44371.0
..................
2024-10-18 19:57:24-04:00584.80584.80584.80584.80100.0
2024-10-18 19:57:48-04:00584.84584.84584.84584.84622.0
2024-10-18 19:58:48-04:00584.77584.79584.77584.794158.0
2024-10-18 19:59:36-04:00584.80584.82584.80584.82298.0
2024-10-18 19:59:48-04:00584.76584.76584.72584.72258.0
\n", "

64218 rows × 5 columns

\n", "
" ], "text/plain": [ " open high low close volume\n", "time \n", "2024-09-16 04:01:24-04:00 562.22 562.22 562.22 562.22 200.0\n", "2024-09-16 04:02:24-04:00 562.17 562.17 562.17 562.17 293.0\n", "2024-09-16 04:04:36-04:00 562.54 562.54 562.54 562.54 100.0\n", "2024-09-16 04:10:00-04:00 562.39 562.39 562.39 562.39 102.0\n", "2024-09-16 04:10:24-04:00 562.44 562.44 562.44 562.44 371.0\n", "... ... ... ... ... ...\n", "2024-10-18 19:57:24-04:00 584.80 584.80 584.80 584.80 100.0\n", "2024-10-18 19:57:48-04:00 584.84 584.84 584.84 584.84 622.0\n", "2024-10-18 19:58:48-04:00 584.77 584.79 584.77 584.79 4158.0\n", "2024-10-18 19:59:36-04:00 584.80 584.82 584.80 584.82 298.0\n", "2024-10-18 19:59:48-04:00 584.76 584.76 584.72 584.72 258.0\n", "\n", "[64218 rows x 5 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#This is how to call LOAD function\n", "symbol = [\"SPY\"]\n", "#datetime in zoneNY \n", "day_start = datetime(2024, 9, 15, 9, 30, 0)\n", "day_stop = datetime(2024, 10, 20, 16, 0, 0)\n", "day_start = zoneNY.localize(day_start)\n", "day_stop = zoneNY.localize(day_stop)\n", "\n", "#requested AGG\n", "resolution = 12 #12s bars\n", "agg_type = AggType.OHLCV #other types AggType.OHLCV_VOL, AggType.OHLCV_DOL, AggType.OHLCV_RENKO\n", "exclude_conditions = ['C','O','4','B','7','V','P','W','U','Z','F','9','M','6'] #None to defaults\n", "minsize = 100 #min trade size to include\n", "main_session_only = False\n", "force_remote = False\n", "\n", "data = load_data(symbol = symbol,\n", " agg_type = agg_type,\n", " resolution = resolution,\n", " start_date = day_start,\n", " end_date = day_stop,\n", " #exclude_conditions = None,\n", " minsize = minsize,\n", " main_session_only = main_session_only,\n", " force_remote = force_remote,\n", " return_vbt = True, #returns vbt object\n", " verbose = False\n", " )\n", "data.ohlcv.data[symbol[0]]\n", "#data.ohlcv.data[symbol[0]].lw.plot()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "data.ohlcv.data[symbol[0]]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prepare daily trade cache\n", "This is how to prepare trade cache for given symbol and period (if daily trades are not cached they are remotely fetched.)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "symbols = [\"BAC\", \"AAPL\"]\n", "#datetime in zoneNY \n", "day_start = datetime(2024, 10, 1, 9, 45, 0)\n", "day_stop = datetime(2024, 10, 27, 15, 1, 0)\n", "day_start = zoneNY.localize(day_start)\n", "day_stop = zoneNY.localize(day_stop)\n", "force_remote = False\n", "\n", "prepare_trade_cache(symbols, day_start, day_stop, force_remote, verbose = True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Prepare daily trade cache - cli script\n", "\n", "Python script prepares trade cache for specified symbols and date range.\n", "\n", "Usually 1 day takes about 35s. It is stored in /tradescache/ directory as daily file keyed by symbol.\n", "\n", "To run this script in the background with specific arguments:\n", "\n", "```bash\n", "# Running without forcing remote fetch\n", "python3 prepare_cache.py --symbols BAC AAPL --day_start 2024-10-14 --day_stop 2024-10-18 &\n", "\n", "# Running with force_remote set to True\n", "python3 prepare_cache.py --symbols BAC AAPL --day_start 2024-10-14 --day_stop 2024-10-18 --force_remote &\n", "\n", "```" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Aggregated data are stored per symbol, date range and conditions. If requested dates are matched with existing stored data with same conditions but wider data spans they are loaded from this file.\n", "\n", "This is the matching part:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from ttools.utils import list_matching_files, print_matching_files_info, zoneNY\n", "from datetime import datetime\n", "from ttools.config import AGG_CACHE\n", "\n", "# Find all files covering January 15, 2024 9:30 to 16:00\n", "files = list_matching_files(\n", " symbol='SPY',\n", " resolution=\"1\",\n", " agg_type='AggType.OHLCV',\n", " start_date=datetime(2024, 1, 15, 9, 30),\n", " end_date=datetime(2024, 1, 15, 16, 0)\n", ")\n", "\n", "#print_matching_files_info(files)\n", "\n", "# Example with all parameters specified\n", "specific_files = list_matching_files(\n", " symbol=\"SPY\",\n", " agg_type=\"AggType.OHLCV\",\n", " resolution=\"12\",\n", " start_date=zoneNY.localize(datetime(2024, 1, 15, 9, 30)),\n", " end_date=zoneNY.localize(datetime(2024, 1, 15, 16, 0)),\n", " excludes_str=\"4679BCFMOPUVWZ\",\n", " minsize=100,\n", " main_session_only=True\n", ")\n", "\n", "print_matching_files_info(specific_files)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "From this file the subset of dates are loaded. Usually this is all done automatically by `load_data` in loader." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#loading manually range subset from existing files\n", "start = zoneNY.localize(datetime(2024, 1, 15, 9, 30))\n", "end = zoneNY.localize(datetime(2024, 10, 20, 16, 00))\n", "\n", "ohlcv_df = pd.read_parquet(\n", " AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-01-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-True.parquet\", \n", " engine='pyarrow',\n", " filters=[('time', '>=', start), \n", " ('time', '<=', end)]\n", ")\n", "\n", "ohlcv_df" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TTOOLS: Loaded env variables from file /Users/davidbrazda/Documents/Development/python/.env\n" ] } ], "source": [ "\n", "from ttools.loaders import fetch_daily_stock_trades, fetch_trades_parallel\n", "from ttools.utils import zoneNY\n", "from datetime import datetime" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Fetching trades for whole range" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "SPY Contains 46 market days\n", "SPY All 46 split files loaded in 10.521624088287354 seconds\n", "Trimming 2024-01-16 09:30:00-05:00 2024-03-20 16:00:00-04:00\n", "excluding ['C', 'O', '4', 'B', '7', 'V', 'P', 'W', 'U', 'Z', 'F', '9', 'M', '6']\n", "exclude done\n", "minsize 100\n", "minsize done\n", "SPY filtered\n", "\n", "DatetimeIndex: 6513606 entries, 2024-01-16 09:30:00.001443-05:00 to 2024-03-20 15:59:59.992808-04:00\n", "Data columns (total 6 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 x object \n", " 1 p float64\n", " 2 s int64 \n", " 3 i int64 \n", " 4 c object \n", " 5 z object \n", "dtypes: float64(1), int64(2), object(3)\n", "memory usage: 347.9+ MB\n" ] } ], "source": [ "\n", "\n", "#fethcing one day\n", "# df = fetch_daily_stock_trades(symbol=\"SPY\",\n", "# start=zoneNY.localize(datetime(2024, 1, 16, 9, 30)),\n", "# end=zoneNY.localize(datetime(2024, 1, 16, 16, 00)))\n", "# df.info()\n", "\n", "#fetching multiple days with parallel\n", "df = fetch_trades_parallel(symbol=\"SPY\",\n", " start_date=zoneNY.localize(datetime(2024, 1, 16, 9, 30)),\n", " end_date=zoneNY.localize(datetime(2024, 3, 20, 16, 00)))\n", "\n", "df.info()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "df.info()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#comparing dataframes\n", "from ttools.utils import AGG_CACHE, compare_dataframes\n", "import pandas as pd\n", "file1 = AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-02-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-False.parquet\"\n", "file2 = AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-02-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-False_older2.parquet\"\n", "df1 = pd.read_parquet(file1)\n", "df2 = pd.read_parquet(file2)\n", "df1.equals(df2)\n", "\n", "#compare_dataframes(df1, df2)" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.11" } }, "nbformat": 4, "nbformat_minor": 2 }