{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Load data\n", "\n", "Make sure you have .env file in ttools or any parent dir with your Alpaca keys.\n", "\n", "```bash\n", "ACCOUNT1_LIVE_API_KEY=api_key\n", "ACCOUNT1_LIVE_SECRET_KEY=secret_key\n", "```\n", "\n", "### Cache directories\n", "\n", "Daily trade files - DATADIR/tradecache\n", "Agg data cache - DATADIR/aggcache\n", "\n", "`DATADIR` - user_data_dir from appdirs library - see `config.py`" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TTOOLS: Loaded env variables from file /Users/davidbrazda/Documents/Development/python/.env\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "from ttools.utils import AggType\n", "from datetime import datetime\n", "from ttools.aggregator_vectorized import generate_time_bars_nb, aggregate_trades\n", "from ttools.loaders import load_data, prepare_trade_cache\n", "from ttools.utils import zoneNY\n", "import vectorbtpro as vbt\n", "from lightweight_charts import PlotDFAccessor, PlotSRAccessor\n", "\n", "\n", "vbt.settings.set_theme(\"dark\")\n", "vbt.settings['plotting']['layout']['width'] = 1280\n", "vbt.settings.plotting.auto_rangebreaks = True\n", "# Set the option to display with pagination\n", "pd.set_option('display.notebook_repr_html', True)\n", "pd.set_option('display.max_rows', 10) # Number of rows per page\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Fetching aggregated data\n", "\n", "Available aggregation types: \n", "- time based bars - AggType.OHLCV\n", "- volume based bars - AggType.OHLCV_VOL, resolution = volume threshold\n", "- dollar based bars - AggType.OHLCV_DOL, resolution = dollar threshold\n", "- renko bars - AggType.OHLCV_RENKO resolution = bricksize" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
openhighlowclosevolume
time
2024-02-15 09:30:00-05:00499.29499.41499.2900499.3200161900.0
2024-02-15 09:30:01-05:00499.32499.41499.3000499.400010900.0
2024-02-15 09:30:02-05:00499.36499.40499.3550499.38007040.0
2024-02-15 09:30:03-05:00499.39499.42499.3800499.40008717.0
2024-02-15 09:30:04-05:00499.40499.40499.3500499.35003265.0
..................
2024-03-18 15:59:55-04:00512.94512.94512.8600512.89007345.0
2024-03-18 15:59:56-04:00512.90512.90512.8700512.88002551.0
2024-03-18 15:59:57-04:00512.89512.91512.8500512.870118063.0
2024-03-18 15:59:58-04:00512.87512.90512.8496512.90007734.0
2024-03-18 15:59:59-04:00512.92512.92512.8200512.870037159.0
\n", "

417345 rows × 5 columns

\n", "
" ], "text/plain": [ " open high low close volume\n", "time \n", "2024-02-15 09:30:00-05:00 499.29 499.41 499.2900 499.3200 161900.0\n", "2024-02-15 09:30:01-05:00 499.32 499.41 499.3000 499.4000 10900.0\n", "2024-02-15 09:30:02-05:00 499.36 499.40 499.3550 499.3800 7040.0\n", "2024-02-15 09:30:03-05:00 499.39 499.42 499.3800 499.4000 8717.0\n", "2024-02-15 09:30:04-05:00 499.40 499.40 499.3500 499.3500 3265.0\n", "... ... ... ... ... ...\n", "2024-03-18 15:59:55-04:00 512.94 512.94 512.8600 512.8900 7345.0\n", "2024-03-18 15:59:56-04:00 512.90 512.90 512.8700 512.8800 2551.0\n", "2024-03-18 15:59:57-04:00 512.89 512.91 512.8500 512.8701 18063.0\n", "2024-03-18 15:59:58-04:00 512.87 512.90 512.8496 512.9000 7734.0\n", "2024-03-18 15:59:59-04:00 512.92 512.92 512.8200 512.8700 37159.0\n", "\n", "[417345 rows x 5 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#This is how to call LOAD function\n", "symbol = [\"SPY\"]\n", "#datetime in zoneNY \n", "day_start = datetime(2024, 2, 15, 9, 30, 0)\n", "day_stop = datetime(2024, 3, 18, 16, 0, 0)\n", "day_start = zoneNY.localize(day_start)\n", "day_stop = zoneNY.localize(day_stop)\n", "\n", "#requested AGG\n", "resolution = 1 #12s bars\n", "agg_type = AggType.OHLCV #other types AggType.OHLCV_VOL, AggType.OHLCV_DOL, AggType.OHLCV_RENKO\n", "exclude_conditions = ['C','O','4','B','7','V','P','W','U','Z','F','9','M','6'] #None to defaults\n", "minsize = 100 #min trade size to include\n", "main_session_only = True\n", "force_remote = False\n", "\n", "data = load_data(symbol = symbol,\n", " agg_type = agg_type,\n", " resolution = resolution,\n", " start_date = day_start,\n", " end_date = day_stop,\n", " #exclude_conditions = None,\n", " minsize = minsize,\n", " main_session_only = main_session_only,\n", " force_remote = force_remote,\n", " return_vbt = True, #returns vbt object\n", " verbose = False\n", " )\n", "data.ohlcv.data[symbol[0]]\n", "#data.ohlcv.data[symbol[0]].lw.plot()\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
openhighlowclosevolume
time
2024-10-14 09:45:00-04:0041.965041.97041.95041.950017895.0
2024-10-14 09:45:12-04:0041.958941.96541.95041.96506281.0
2024-10-14 09:45:24-04:0041.965042.00541.96541.99753522.0
2024-10-14 09:45:36-04:0041.990042.00541.99042.00005960.0
2024-10-14 09:45:48-04:0042.005042.04042.00542.03009113.0
..................
2024-10-16 15:00:00-04:0042.915042.91542.91042.910012872.0
2024-10-16 15:00:12-04:0042.915042.92042.91042.92007574.0
2024-10-16 15:00:24-04:0042.920042.92042.91042.92001769.0
2024-10-16 15:00:36-04:0042.920042.92042.90542.905026599.0
2024-10-16 15:00:48-04:0042.905042.90542.88042.88009216.0
\n", "

5480 rows × 5 columns

\n", "
" ], "text/plain": [ " open high low close volume\n", "time \n", "2024-10-14 09:45:00-04:00 41.9650 41.970 41.950 41.9500 17895.0\n", "2024-10-14 09:45:12-04:00 41.9589 41.965 41.950 41.9650 6281.0\n", "2024-10-14 09:45:24-04:00 41.9650 42.005 41.965 41.9975 3522.0\n", "2024-10-14 09:45:36-04:00 41.9900 42.005 41.990 42.0000 5960.0\n", "2024-10-14 09:45:48-04:00 42.0050 42.040 42.005 42.0300 9113.0\n", "... ... ... ... ... ...\n", "2024-10-16 15:00:00-04:00 42.9150 42.915 42.910 42.9100 12872.0\n", "2024-10-16 15:00:12-04:00 42.9150 42.920 42.910 42.9200 7574.0\n", "2024-10-16 15:00:24-04:00 42.9200 42.920 42.910 42.9200 1769.0\n", "2024-10-16 15:00:36-04:00 42.9200 42.920 42.905 42.9050 26599.0\n", "2024-10-16 15:00:48-04:00 42.9050 42.905 42.880 42.8800 9216.0\n", "\n", "[5480 rows x 5 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.ohlcv.data[symbol[0]]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prepare daily trade cache\n", "This is how to prepare trade cache for given symbol and period (if daily trades are not cached they are remotely fetched.)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "symbols = [\"BAC\", \"AAPL\"]\n", "#datetime in zoneNY \n", "day_start = datetime(2024, 10, 1, 9, 45, 0)\n", "day_stop = datetime(2024, 10, 27, 15, 1, 0)\n", "day_start = zoneNY.localize(day_start)\n", "day_stop = zoneNY.localize(day_stop)\n", "force_remote = False\n", "\n", "prepare_trade_cache(symbols, day_start, day_stop, force_remote, verbose = True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Prepare daily trade cache - cli script\n", "\n", "Python script prepares trade cache for specified symbols and date range.\n", "\n", "Usually 1 day takes about 35s. It is stored in /tradescache/ directory as daily file keyed by symbol.\n", "\n", "To run this script in the background with specific arguments:\n", "\n", "```bash\n", "# Running without forcing remote fetch\n", "python3 prepare_cache.py --symbols BAC AAPL --day_start 2024-10-14 --day_stop 2024-10-18 &\n", "\n", "# Running with force_remote set to True\n", "python3 prepare_cache.py --symbols BAC AAPL --day_start 2024-10-14 --day_stop 2024-10-18 --force_remote &\n", "\n", "```" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Aggregated data are stored per symbol, date range and conditions. If requested dates are matched with existing stored data with same conditions but wider data spans they are loaded from this file.\n", "\n", "This is the matching part:" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "File: SPY-AggType.OHLCV-12-2024-01-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-True.parquet\n", "Coverage: 2024-01-15 09:30:00 to 2024-10-20 16:00:00\n", "Symbol: SPY\n", "Agg Type: AggType.OHLCV\n", "Resolution: 12\n", "Excludes: 4679BCFMOPUVWZ\n", "Minsize: 100\n", "Main Session Only: True\n", "--------------------------------------------------------------------------------\n" ] } ], "source": [ "from ttools.utils import list_matching_files, print_matching_files_info, zoneNY\n", "from datetime import datetime\n", "from ttools.config import AGG_CACHE\n", "\n", "# Find all files covering January 15, 2024 9:30 to 16:00\n", "files = list_matching_files(\n", " symbol='SPY',\n", " resolution=\"1\",\n", " agg_type='AggType.OHLCV',\n", " start_date=datetime(2024, 1, 15, 9, 30),\n", " end_date=datetime(2024, 1, 15, 16, 0)\n", ")\n", "\n", "#print_matching_files_info(files)\n", "\n", "# Example with all parameters specified\n", "specific_files = list_matching_files(\n", " symbol=\"SPY\",\n", " agg_type=\"AggType.OHLCV\",\n", " resolution=\"12\",\n", " start_date=zoneNY.localize(datetime(2024, 1, 15, 9, 30)),\n", " end_date=zoneNY.localize(datetime(2024, 1, 15, 16, 0)),\n", " excludes_str=\"4679BCFMOPUVWZ\",\n", " minsize=100,\n", " main_session_only=True\n", ")\n", "\n", "print_matching_files_info(specific_files)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And date subset loaded from parquet. Usually this is all done yb `load_data` in loader." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
openhighlowclosevolumetradesupdatedvwapbuyvolumesellvolume
time
2024-01-16 09:30:00-05:00475.250475.3600475.20475.285255386.093.02024-01-16 09:30:01.002183-05:00475.2517253692.0242756.0
2024-01-16 09:30:01-05:00475.335475.3350475.23475.26015161.0100.02024-01-16 09:30:02.007313-05:00475.2833904386.04944.0
2024-01-16 09:30:02-05:00475.250475.3000475.24475.3006993.039.02024-01-16 09:30:03.008912-05:00475.2625071900.02256.0
2024-01-16 09:30:03-05:00475.290475.3200475.24475.2708497.047.02024-01-16 09:30:04.201093-05:00475.2752801300.03200.0
2024-01-16 09:30:04-05:00475.250475.2700475.22475.2705367.037.02024-01-16 09:30:05.004980-05:00475.2343531613.01247.0
.................................
2024-10-18 15:59:55-04:00584.520584.5800584.51584.58010357.047.02024-10-18 15:59:56.008928-04:00584.5438701600.01100.0
2024-10-18 15:59:56-04:00584.570584.6091584.55584.5506527.032.02024-10-18 15:59:57.007658-04:00584.5666431525.01002.0
2024-10-18 15:59:57-04:00584.560584.6100584.56584.6005068.023.02024-10-18 15:59:58.000435-04:00584.5962491960.0900.0
2024-10-18 15:59:58-04:00584.590584.6200584.56584.5608786.023.02024-10-18 15:59:59.041984-04:00584.5922172859.03921.0
2024-10-18 15:59:59-04:00584.560584.6100584.56584.57012583.069.02024-10-18 15:59:59.982132-04:00584.5831315303.01980.0
\n", "

3384529 rows × 10 columns

\n", "
" ], "text/plain": [ " open high low close volume \\\n", "time \n", "2024-01-16 09:30:00-05:00 475.250 475.3600 475.20 475.285 255386.0 \n", "2024-01-16 09:30:01-05:00 475.335 475.3350 475.23 475.260 15161.0 \n", "2024-01-16 09:30:02-05:00 475.250 475.3000 475.24 475.300 6993.0 \n", "2024-01-16 09:30:03-05:00 475.290 475.3200 475.24 475.270 8497.0 \n", "2024-01-16 09:30:04-05:00 475.250 475.2700 475.22 475.270 5367.0 \n", "... ... ... ... ... ... \n", "2024-10-18 15:59:55-04:00 584.520 584.5800 584.51 584.580 10357.0 \n", "2024-10-18 15:59:56-04:00 584.570 584.6091 584.55 584.550 6527.0 \n", "2024-10-18 15:59:57-04:00 584.560 584.6100 584.56 584.600 5068.0 \n", "2024-10-18 15:59:58-04:00 584.590 584.6200 584.56 584.560 8786.0 \n", "2024-10-18 15:59:59-04:00 584.560 584.6100 584.56 584.570 12583.0 \n", "\n", " trades updated \\\n", "time \n", "2024-01-16 09:30:00-05:00 93.0 2024-01-16 09:30:01.002183-05:00 \n", "2024-01-16 09:30:01-05:00 100.0 2024-01-16 09:30:02.007313-05:00 \n", "2024-01-16 09:30:02-05:00 39.0 2024-01-16 09:30:03.008912-05:00 \n", "2024-01-16 09:30:03-05:00 47.0 2024-01-16 09:30:04.201093-05:00 \n", "2024-01-16 09:30:04-05:00 37.0 2024-01-16 09:30:05.004980-05:00 \n", "... ... ... \n", "2024-10-18 15:59:55-04:00 47.0 2024-10-18 15:59:56.008928-04:00 \n", "2024-10-18 15:59:56-04:00 32.0 2024-10-18 15:59:57.007658-04:00 \n", "2024-10-18 15:59:57-04:00 23.0 2024-10-18 15:59:58.000435-04:00 \n", "2024-10-18 15:59:58-04:00 23.0 2024-10-18 15:59:59.041984-04:00 \n", "2024-10-18 15:59:59-04:00 69.0 2024-10-18 15:59:59.982132-04:00 \n", "\n", " vwap buyvolume sellvolume \n", "time \n", "2024-01-16 09:30:00-05:00 475.251725 3692.0 242756.0 \n", "2024-01-16 09:30:01-05:00 475.283390 4386.0 4944.0 \n", "2024-01-16 09:30:02-05:00 475.262507 1900.0 2256.0 \n", "2024-01-16 09:30:03-05:00 475.275280 1300.0 3200.0 \n", "2024-01-16 09:30:04-05:00 475.234353 1613.0 1247.0 \n", "... ... ... ... \n", "2024-10-18 15:59:55-04:00 584.543870 1600.0 1100.0 \n", "2024-10-18 15:59:56-04:00 584.566643 1525.0 1002.0 \n", "2024-10-18 15:59:57-04:00 584.596249 1960.0 900.0 \n", "2024-10-18 15:59:58-04:00 584.592217 2859.0 3921.0 \n", "2024-10-18 15:59:59-04:00 584.583131 5303.0 1980.0 \n", "\n", "[3384529 rows x 10 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "start = zoneNY.localize(datetime(2024, 1, 15, 9, 30))\n", "end = zoneNY.localize(datetime(2024, 10, 20, 16, 00))\n", "\n", "ohlcv_df = pd.read_parquet(\n", " AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-01-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-True.parquet\", \n", " engine='pyarrow',\n", " filters=[('time', '>=', start), \n", " ('time', '<=', end)]\n", ")\n", "\n", "ohlcv_df" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.11" } }, "nbformat": 4, "nbformat_minor": 2 }