{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data\n",
    "\n",
    "Make sure you have .env file in ttools or any parent dir with your Alpaca keys.\n",
    "\n",
    "```bash\n",
    "ACCOUNT1_LIVE_API_KEY=api_key\n",
    "ACCOUNT1_LIVE_SECRET_KEY=secret_key\n",
    "```\n",
    "\n",
    "###  Cache directories\n",
    "\n",
    "Daily trade files - DATADIR/tradecache\n",
    "Agg data cache - DATADIR/aggcache\n",
    "\n",
    "`DATADIR` - user_data_dir from appdirs library - see `config.py`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TTOOLS: Loaded env variables from file /Users/davidbrazda/Documents/Development/python/.env\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from ttools.utils import AggType\n",
    "from datetime import datetime\n",
    "from ttools.aggregator_vectorized import generate_time_bars_nb, aggregate_trades\n",
    "from ttools.loaders import load_data, prepare_trade_cache, fetch_daily_stock_trades\n",
    "from ttools.utils import zoneNY\n",
    "import vectorbtpro as vbt\n",
    "from lightweight_charts import PlotDFAccessor, PlotSRAccessor\n",
    "\n",
    "\n",
    "vbt.settings.set_theme(\"dark\")\n",
    "vbt.settings['plotting']['layout']['width'] = 1280\n",
    "vbt.settings.plotting.auto_rangebreaks = True\n",
    "# Set the option to display with pagination\n",
    "pd.set_option('display.notebook_repr_html', True)\n",
    "pd.set_option('display.max_rows', 10)  # Number of rows per page\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fetching aggregated data\n",
    "\n",
    "Available aggregation types: \n",
    "- time based bars - AggType.OHLCV\n",
    "- volume based bars - AggType.OHLCV_VOL, resolution = volume threshold\n",
    "- dollar based bars - AggType.OHLCV_DOL, resolution = dollar threshold\n",
    "- renko bars - AggType.OHLCV_RENKO resolution = bricksize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2024-09-16 04:01:24-04:00</th>\n",
       "      <td>562.22</td>\n",
       "      <td>562.22</td>\n",
       "      <td>562.22</td>\n",
       "      <td>562.22</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-09-16 04:02:24-04:00</th>\n",
       "      <td>562.17</td>\n",
       "      <td>562.17</td>\n",
       "      <td>562.17</td>\n",
       "      <td>562.17</td>\n",
       "      <td>293.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-09-16 04:04:36-04:00</th>\n",
       "      <td>562.54</td>\n",
       "      <td>562.54</td>\n",
       "      <td>562.54</td>\n",
       "      <td>562.54</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-09-16 04:10:00-04:00</th>\n",
       "      <td>562.39</td>\n",
       "      <td>562.39</td>\n",
       "      <td>562.39</td>\n",
       "      <td>562.39</td>\n",
       "      <td>102.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-09-16 04:10:24-04:00</th>\n",
       "      <td>562.44</td>\n",
       "      <td>562.44</td>\n",
       "      <td>562.44</td>\n",
       "      <td>562.44</td>\n",
       "      <td>371.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-10-18 19:57:24-04:00</th>\n",
       "      <td>584.80</td>\n",
       "      <td>584.80</td>\n",
       "      <td>584.80</td>\n",
       "      <td>584.80</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-10-18 19:57:48-04:00</th>\n",
       "      <td>584.84</td>\n",
       "      <td>584.84</td>\n",
       "      <td>584.84</td>\n",
       "      <td>584.84</td>\n",
       "      <td>622.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-10-18 19:58:48-04:00</th>\n",
       "      <td>584.77</td>\n",
       "      <td>584.79</td>\n",
       "      <td>584.77</td>\n",
       "      <td>584.79</td>\n",
       "      <td>4158.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-10-18 19:59:36-04:00</th>\n",
       "      <td>584.80</td>\n",
       "      <td>584.82</td>\n",
       "      <td>584.80</td>\n",
       "      <td>584.82</td>\n",
       "      <td>298.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-10-18 19:59:48-04:00</th>\n",
       "      <td>584.76</td>\n",
       "      <td>584.76</td>\n",
       "      <td>584.72</td>\n",
       "      <td>584.72</td>\n",
       "      <td>258.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64218 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             open    high     low   close  volume\n",
       "time                                                             \n",
       "2024-09-16 04:01:24-04:00  562.22  562.22  562.22  562.22   200.0\n",
       "2024-09-16 04:02:24-04:00  562.17  562.17  562.17  562.17   293.0\n",
       "2024-09-16 04:04:36-04:00  562.54  562.54  562.54  562.54   100.0\n",
       "2024-09-16 04:10:00-04:00  562.39  562.39  562.39  562.39   102.0\n",
       "2024-09-16 04:10:24-04:00  562.44  562.44  562.44  562.44   371.0\n",
       "...                           ...     ...     ...     ...     ...\n",
       "2024-10-18 19:57:24-04:00  584.80  584.80  584.80  584.80   100.0\n",
       "2024-10-18 19:57:48-04:00  584.84  584.84  584.84  584.84   622.0\n",
       "2024-10-18 19:58:48-04:00  584.77  584.79  584.77  584.79  4158.0\n",
       "2024-10-18 19:59:36-04:00  584.80  584.82  584.80  584.82   298.0\n",
       "2024-10-18 19:59:48-04:00  584.76  584.76  584.72  584.72   258.0\n",
       "\n",
       "[64218 rows x 5 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This is how to call LOAD function\n",
    "symbol = [\"SPY\"]\n",
    "#datetime in zoneNY \n",
    "day_start = datetime(2024, 9, 15, 9, 30, 0)\n",
    "day_stop = datetime(2024, 10, 20, 16, 0, 0)\n",
    "day_start = zoneNY.localize(day_start)\n",
    "day_stop = zoneNY.localize(day_stop)\n",
    "\n",
    "#requested AGG\n",
    "resolution = 12 #12s bars\n",
    "agg_type = AggType.OHLCV #other types AggType.OHLCV_VOL, AggType.OHLCV_DOL, AggType.OHLCV_RENKO\n",
    "exclude_conditions = ['C','O','4','B','7','V','P','W','U','Z','F','9','M','6'] #None to defaults\n",
    "minsize = 100 #min trade size to include\n",
    "main_session_only = False\n",
    "force_remote = False\n",
    "\n",
    "data = load_data(symbol = symbol,\n",
    "                     agg_type = agg_type,\n",
    "                     resolution = resolution,\n",
    "                     start_date = day_start,\n",
    "                     end_date = day_stop,\n",
    "                     #exclude_conditions = None,\n",
    "                     minsize = minsize,\n",
    "                     main_session_only = main_session_only,\n",
    "                     force_remote = force_remote,\n",
    "                     return_vbt = True, #returns vbt object\n",
    "                     verbose = False\n",
    "                     )\n",
    "data.ohlcv.data[symbol[0]]\n",
    "#data.ohlcv.data[symbol[0]].lw.plot()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.ohlcv.data[symbol[0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare daily trade cache\n",
    "This is how to prepare trade cache for given symbol and period (if daily trades are not cached they are remotely fetched.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "symbols = [\"BAC\", \"AAPL\"]\n",
    "#datetime in zoneNY \n",
    "day_start = datetime(2024, 10, 1, 9, 45, 0)\n",
    "day_stop = datetime(2024, 10, 27, 15, 1, 0)\n",
    "day_start = zoneNY.localize(day_start)\n",
    "day_stop = zoneNY.localize(day_stop)\n",
    "force_remote = False\n",
    "\n",
    "prepare_trade_cache(symbols, day_start, day_stop, force_remote, verbose = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare daily trade cache - cli script\n",
    "\n",
    "Python script prepares trade cache for specified symbols and date range.\n",
    "\n",
    "Usually 1 day takes about 35s. It is stored in /tradescache/ directory as daily file keyed by symbol.\n",
    "\n",
    "To run this script in the background with specific arguments:\n",
    "\n",
    "```bash\n",
    "# Running without forcing remote fetch\n",
    "python3 prepare_cache.py --symbols BAC AAPL --day_start 2024-10-14 --day_stop 2024-10-18 &\n",
    "\n",
    "# Running with force_remote set to True\n",
    "python3 prepare_cache.py --symbols BAC AAPL --day_start 2024-10-14 --day_stop 2024-10-18 --force_remote &\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aggregated data are stored per symbol, date range and conditions. If requested dates are matched with existing stored data with same conditions but wider data spans they are loaded from this file.\n",
    "\n",
    "This is the matching part:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ttools.utils import list_matching_files, print_matching_files_info, zoneNY\n",
    "from datetime import datetime\n",
    "from ttools.config import AGG_CACHE\n",
    "\n",
    "# Find all files covering January 15, 2024 9:30 to 16:00\n",
    "files = list_matching_files(\n",
    "    symbol='SPY',\n",
    "    resolution=\"1\",\n",
    "    agg_type='AggType.OHLCV',\n",
    "    start_date=datetime(2024, 1, 15, 9, 30),\n",
    "    end_date=datetime(2024, 1, 15, 16, 0)\n",
    ")\n",
    "\n",
    "#print_matching_files_info(files)\n",
    "\n",
    "# Example with all parameters specified\n",
    "specific_files = list_matching_files(\n",
    "    symbol=\"SPY\",\n",
    "    agg_type=\"AggType.OHLCV\",\n",
    "    resolution=\"12\",\n",
    "    start_date=zoneNY.localize(datetime(2024, 1, 15, 9, 30)),\n",
    "    end_date=zoneNY.localize(datetime(2024, 1, 15, 16, 0)),\n",
    "    excludes_str=\"4679BCFMOPUVWZ\",\n",
    "    minsize=100,\n",
    "    main_session_only=True\n",
    ")\n",
    "\n",
    "print_matching_files_info(specific_files)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From this file the subset of dates are loaded. Usually this is all done automatically by `load_data` in loader."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'zoneNY' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#loading manually range subset from existing files\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m start \u001b[38;5;241m=\u001b[39m \u001b[43mzoneNY\u001b[49m\u001b[38;5;241m.\u001b[39mlocalize(datetime(\u001b[38;5;241m2024\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m15\u001b[39m, \u001b[38;5;241m9\u001b[39m, \u001b[38;5;241m30\u001b[39m))\n\u001b[1;32m      3\u001b[0m end \u001b[38;5;241m=\u001b[39m zoneNY\u001b[38;5;241m.\u001b[39mlocalize(datetime(\u001b[38;5;241m2024\u001b[39m, \u001b[38;5;241m10\u001b[39m, \u001b[38;5;241m20\u001b[39m, \u001b[38;5;241m16\u001b[39m, \u001b[38;5;241m00\u001b[39m))\n\u001b[1;32m      5\u001b[0m ohlcv_df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mread_parquet(\n\u001b[1;32m      6\u001b[0m     AGG_CACHE \u001b[38;5;241m/\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSPY-AggType.OHLCV-1-2024-01-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-True.parquet\u001b[39m\u001b[38;5;124m\"\u001b[39m, \n\u001b[1;32m      7\u001b[0m     engine\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpyarrow\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m      8\u001b[0m     filters\u001b[38;5;241m=\u001b[39m[(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtime\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m>=\u001b[39m\u001b[38;5;124m'\u001b[39m, start), \n\u001b[1;32m      9\u001b[0m             (\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtime\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m<=\u001b[39m\u001b[38;5;124m'\u001b[39m, end)]\n\u001b[1;32m     10\u001b[0m )\n",
      "\u001b[0;31mNameError\u001b[0m: name 'zoneNY' is not defined"
     ]
    }
   ],
   "source": [
    "#loading manually range subset from existing files\n",
    "start = zoneNY.localize(datetime(2024, 1, 15, 9, 30))\n",
    "end = zoneNY.localize(datetime(2024, 10, 20, 16, 00))\n",
    "\n",
    "ohlcv_df = pd.read_parquet(\n",
    "    AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-01-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-True.parquet\", \n",
    "    engine='pyarrow',\n",
    "    filters=[('time', '>=', start), \n",
    "            ('time', '<=', end)]\n",
    ")\n",
    "\n",
    "ohlcv_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TTOOLS: Loaded env variables from file /Users/davidbrazda/Documents/Development/python/.env\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from ttools.loaders import fetch_daily_stock_trades, fetch_trades_parallel\n",
    "from ttools.utils import zoneNY\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fetching trades for whole range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BAC Contains 1  market days\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "BAC Remote fetching: 100%|██████████| 1/1 [00:00<00:00, 434.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching from remote.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "BAC Receiving trades:   0%|          | 0/1 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remote fetched completed whole day 2024-01-16\n",
      "Exact UTC range fetched: 2024-01-16 05:00:00+00:00 - 2024-01-17 04:59:59.999999+00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "BAC Receiving trades: 100%|██████████| 1/1 [00:42<00:00, 42.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to CACHE /Users/davidbrazda/Library/Application Support/v2realbot/tradecache/BAC-2024-01-16.parquet\n",
      "Trimming 2024-01-16 00:00:00-05:00 2024-01-16 23:59:00-05:00\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 222754 entries, 2024-01-16 04:00:00.009225-05:00 to 2024-01-16 19:59:48.834830-05:00\n",
      "Data columns (total 6 columns):\n",
      " #   Column  Non-Null Count   Dtype  \n",
      "---  ------  --------------   -----  \n",
      " 0   x       222754 non-null  object \n",
      " 1   p       222754 non-null  float64\n",
      " 2   s       222754 non-null  int64  \n",
      " 3   i       222754 non-null  int64  \n",
      " 4   c       222754 non-null  object \n",
      " 5   z       222754 non-null  object \n",
      "dtypes: float64(1), int64(2), object(3)\n",
      "memory usage: 11.9+ MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "#fethcing one day\n",
    "# df = fetch_daily_stock_trades(symbol=\"SPY\",\n",
    "#                               start=zoneNY.localize(datetime(2024, 1, 16, 9, 30)),\n",
    "#                               end=zoneNY.localize(datetime(2024, 1, 16, 16, 00)))\n",
    "# df.info()\n",
    "\n",
    "#fetching multiple days with parallel\n",
    "df = fetch_trades_parallel(symbol=\"BAC\",\n",
    "                              start_date=zoneNY.localize(datetime(2024, 1, 16, 0, 0)),\n",
    "                              end_date=zoneNY.localize(datetime(2024, 1, 16, 23, 59)),\n",
    "                              main_session_only=False,\n",
    "                              exclude_conditions=None,\n",
    "                              minsize=None,\n",
    "                              force_remote=True)\n",
    "\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>p</th>\n",
       "      <th>s</th>\n",
       "      <th>i</th>\n",
       "      <th>c</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:00.009225-05:00</th>\n",
       "      <td>K</td>\n",
       "      <td>32.800</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027912</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:00.012088-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.580</td>\n",
       "      <td>8</td>\n",
       "      <td>52983525027890</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:02.299262-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.750</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027916</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:03.895322-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.640</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027920</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:04.145553-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.740</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027921</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 18:58:10.081270-05:00</th>\n",
       "      <td>D</td>\n",
       "      <td>32.104</td>\n",
       "      <td>10</td>\n",
       "      <td>79371957716549</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 18:58:11.293971-05:00</th>\n",
       "      <td>T</td>\n",
       "      <td>32.090</td>\n",
       "      <td>3</td>\n",
       "      <td>62883460503386</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 18:58:24.511348-05:00</th>\n",
       "      <td>D</td>\n",
       "      <td>32.110</td>\n",
       "      <td>1</td>\n",
       "      <td>79371957716560</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 18:58:46.648899-05:00</th>\n",
       "      <td>D</td>\n",
       "      <td>32.110</td>\n",
       "      <td>1</td>\n",
       "      <td>79371957716786</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 18:59:54.013894-05:00</th>\n",
       "      <td>D</td>\n",
       "      <td>32.100</td>\n",
       "      <td>1</td>\n",
       "      <td>71710070428229</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159301 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  x       p   s               i          c  z\n",
       "t                                                                            \n",
       "2024-01-16 04:00:00.009225-05:00  K  32.800   1  52983525027912  [ , T, I]  A\n",
       "2024-01-16 04:00:00.012088-05:00  P  32.580   8  52983525027890  [ , T, I]  A\n",
       "2024-01-16 04:00:02.299262-05:00  P  32.750   1  52983525027916  [ , T, I]  A\n",
       "2024-01-16 04:00:03.895322-05:00  P  32.640   1  52983525027920  [ , T, I]  A\n",
       "2024-01-16 04:00:04.145553-05:00  P  32.740   1  52983525027921  [ , T, I]  A\n",
       "...                              ..     ...  ..             ...        ... ..\n",
       "2024-01-16 18:58:10.081270-05:00  D  32.104  10  79371957716549  [ , T, I]  A\n",
       "2024-01-16 18:58:11.293971-05:00  T  32.090   3  62883460503386  [ , T, I]  A\n",
       "2024-01-16 18:58:24.511348-05:00  D  32.110   1  79371957716560  [ , T, I]  A\n",
       "2024-01-16 18:58:46.648899-05:00  D  32.110   1  79371957716786  [ , T, I]  A\n",
       "2024-01-16 18:59:54.013894-05:00  D  32.100   1  71710070428229  [ , T, I]  A\n",
       "\n",
       "[159301 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#comparing dataframes\n",
    "from ttools.utils import AGG_CACHE, compare_dataframes\n",
    "import pandas as pd\n",
    "file1 = AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-02-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-False.parquet\"\n",
    "file2 = AGG_CACHE / \"SPY-AggType.OHLCV-1-2024-02-15T09-30-00-2024-10-20T16-00-00-4679BCFMOPUVWZ-100-False_older2.parquet\"\n",
    "df1 = pd.read_parquet(file1)\n",
    "df2 = pd.read_parquet(file2)\n",
    "df1.equals(df2)\n",
    "\n",
    "#compare_dataframes(df1, df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ttools.config import TRADE_CACHE\n",
    "import pandas as pd\n",
    "file1 = TRADE_CACHE / \"BAC-2024-01-16.parquet\"\n",
    "df1 = pd.read_parquet(file1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>p</th>\n",
       "      <th>s</th>\n",
       "      <th>i</th>\n",
       "      <th>c</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:00.009225-05:00</th>\n",
       "      <td>K</td>\n",
       "      <td>32.80</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027912</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:00.012088-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.58</td>\n",
       "      <td>8</td>\n",
       "      <td>52983525027890</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:00.856156-05:00</th>\n",
       "      <td>K</td>\n",
       "      <td>32.61</td>\n",
       "      <td>14</td>\n",
       "      <td>52983525028705</td>\n",
       "      <td>[ , F, T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:02.299262-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.75</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027916</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 04:00:03.895322-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.64</td>\n",
       "      <td>1</td>\n",
       "      <td>52983525027920</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 19:59:24.796862-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.12</td>\n",
       "      <td>500</td>\n",
       "      <td>52983576997941</td>\n",
       "      <td>[ , T]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 19:59:24.796868-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.12</td>\n",
       "      <td>500</td>\n",
       "      <td>52983576997942</td>\n",
       "      <td>[ , T]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 19:59:24.796868-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.12</td>\n",
       "      <td>500</td>\n",
       "      <td>52983576997943</td>\n",
       "      <td>[ , T]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 19:59:24.796871-05:00</th>\n",
       "      <td>P</td>\n",
       "      <td>32.12</td>\n",
       "      <td>500</td>\n",
       "      <td>52983576997944</td>\n",
       "      <td>[ , T]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024-01-16 19:59:48.834830-05:00</th>\n",
       "      <td>K</td>\n",
       "      <td>32.10</td>\n",
       "      <td>25</td>\n",
       "      <td>52983526941511</td>\n",
       "      <td>[ , T, I]</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>222754 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  x      p    s               i             c  \\\n",
       "t                                                                               \n",
       "2024-01-16 04:00:00.009225-05:00  K  32.80    1  52983525027912     [ , T, I]   \n",
       "2024-01-16 04:00:00.012088-05:00  P  32.58    8  52983525027890     [ , T, I]   \n",
       "2024-01-16 04:00:00.856156-05:00  K  32.61   14  52983525028705  [ , F, T, I]   \n",
       "2024-01-16 04:00:02.299262-05:00  P  32.75    1  52983525027916     [ , T, I]   \n",
       "2024-01-16 04:00:03.895322-05:00  P  32.64    1  52983525027920     [ , T, I]   \n",
       "...                              ..    ...  ...             ...           ...   \n",
       "2024-01-16 19:59:24.796862-05:00  P  32.12  500  52983576997941        [ , T]   \n",
       "2024-01-16 19:59:24.796868-05:00  P  32.12  500  52983576997942        [ , T]   \n",
       "2024-01-16 19:59:24.796868-05:00  P  32.12  500  52983576997943        [ , T]   \n",
       "2024-01-16 19:59:24.796871-05:00  P  32.12  500  52983576997944        [ , T]   \n",
       "2024-01-16 19:59:48.834830-05:00  K  32.10   25  52983526941511     [ , T, I]   \n",
       "\n",
       "                                  z  \n",
       "t                                    \n",
       "2024-01-16 04:00:00.009225-05:00  A  \n",
       "2024-01-16 04:00:00.012088-05:00  A  \n",
       "2024-01-16 04:00:00.856156-05:00  A  \n",
       "2024-01-16 04:00:02.299262-05:00  A  \n",
       "2024-01-16 04:00:03.895322-05:00  A  \n",
       "...                              ..  \n",
       "2024-01-16 19:59:24.796862-05:00  A  \n",
       "2024-01-16 19:59:24.796868-05:00  A  \n",
       "2024-01-16 19:59:24.796868-05:00  A  \n",
       "2024-01-16 19:59:24.796871-05:00  A  \n",
       "2024-01-16 19:59:48.834830-05:00  A  \n",
       "\n",
       "[222754 rows x 6 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}