diff --git a/research/basic.ipynb b/research/basic.ipynb
new file mode 100644
index 0000000..d55e9d3
--- /dev/null
+++ b/research/basic.ipynb
@@ -0,0 +1,144566 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 197,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from v2realbot.tools.loadbatch import load_batch\n",
+ "from v2realbot.utils.utils import zoneNY\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import vectorbtpro as vbt\n",
+ "\n",
+ "vbt.settings.set_theme(\"dark\")\n",
+ "vbt.settings['plotting']['layout']['width'] = 1280\n",
+ "vbt.settings.plotting.auto_rangebreaks = True\n",
+ "# Set the option to display with pagination\n",
+ "pd.set_option('display.notebook_repr_html', True)\n",
+ "pd.set_option('display.max_rows', 10) # Number of rows per page\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 198,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "Resolution : 23\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "Resolution : \u001b[1;36m23\u001b[0m\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "NOTE: DUPLICATES 100/3034 in bars. REMOVING.\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "NOTE: DUPLICATES \u001b[1;36m100\u001b[0m/\u001b[1;36m3034\u001b[0m in bars. REMOVING.\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "Now there are 0/2934\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "Now there are \u001b[1;36m0\u001b[0m/\u001b[1;36m2934\u001b[0m\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "DatetimeIndex: 2934 entries, 2024-03-11 09:30:00-04:00 to 2024-03-13 15:59:51-04:00\n",
+ "Data columns (total 8 columns):\n",
+ " # Column Non-Null Count Dtype \n",
+ "--- ------ -------------- ----- \n",
+ " 0 Open 2934 non-null float64 \n",
+ " 1 High 2934 non-null float64 \n",
+ " 2 Low 2934 non-null float64 \n",
+ " 3 Close 2934 non-null float64 \n",
+ " 4 Volume 2934 non-null int64 \n",
+ " 5 Vwap 2934 non-null float64 \n",
+ " 6 updated 2934 non-null datetime64[ns, US/Eastern]\n",
+ " 7 Rsi14 2934 non-null float64 \n",
+ "dtypes: datetime64[ns, US/Eastern](1), float64(6), int64(1)\n",
+ "memory usage: 270.8 KB\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Open | \n",
+ " High | \n",
+ " Low | \n",
+ " Close | \n",
+ " Volume | \n",
+ " Vwap | \n",
+ " updated | \n",
+ " Rsi14 | \n",
+ "
\n",
+ " \n",
+ " | time | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 2024-03-11 09:30:00-04:00 | \n",
+ " 35.3900 | \n",
+ " 35.440 | \n",
+ " 35.2800 | \n",
+ " 35.380 | \n",
+ " 473759 | \n",
+ " 35.425685 | \n",
+ " 2024-03-11 09:30:23.673966885-04:00 | \n",
+ " 0.0000 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-11 09:30:23-04:00 | \n",
+ " 35.3900 | \n",
+ " 35.490 | \n",
+ " 35.3895 | \n",
+ " 35.475 | \n",
+ " 14119 | \n",
+ " 35.447393 | \n",
+ " 2024-03-11 09:30:46.228593111-04:00 | \n",
+ " 0.0000 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-11 09:30:46-04:00 | \n",
+ " 35.4750 | \n",
+ " 35.480 | \n",
+ " 35.4200 | \n",
+ " 35.460 | \n",
+ " 22169 | \n",
+ " 35.448066 | \n",
+ " 2024-03-11 09:31:09.993794918-04:00 | \n",
+ " 0.0000 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-11 09:31:09-04:00 | \n",
+ " 35.4579 | \n",
+ " 35.540 | \n",
+ " 35.4400 | \n",
+ " 35.535 | \n",
+ " 29984 | \n",
+ " 35.490282 | \n",
+ " 2024-03-11 09:31:32.450469017-04:00 | \n",
+ " 0.0000 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-11 09:31:32-04:00 | \n",
+ " 35.5350 | \n",
+ " 35.595 | \n",
+ " 35.5100 | \n",
+ " 35.590 | \n",
+ " 20348 | \n",
+ " 35.560566 | \n",
+ " 2024-03-11 09:31:55.688961983-04:00 | \n",
+ " 0.0000 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-13 15:58:19-04:00 | \n",
+ " 36.1450 | \n",
+ " 36.155 | \n",
+ " 36.1300 | \n",
+ " 36.145 | \n",
+ " 120386 | \n",
+ " 36.142679 | \n",
+ " 2024-03-13 15:58:42.028856993-04:00 | \n",
+ " 59.6001 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-13 15:58:42-04:00 | \n",
+ " 36.1500 | \n",
+ " 36.150 | \n",
+ " 36.1300 | \n",
+ " 36.130 | \n",
+ " 139068 | \n",
+ " 36.139540 | \n",
+ " 2024-03-13 15:59:05.012445927-04:00 | \n",
+ " 58.2072 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-13 15:59:05-04:00 | \n",
+ " 36.1300 | \n",
+ " 36.140 | \n",
+ " 36.0900 | \n",
+ " 36.090 | \n",
+ " 545827 | \n",
+ " 36.118307 | \n",
+ " 2024-03-13 15:59:28.043910027-04:00 | \n",
+ " 49.7763 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-13 15:59:28-04:00 | \n",
+ " 36.0950 | \n",
+ " 36.110 | \n",
+ " 36.0500 | \n",
+ " 36.080 | \n",
+ " 734354 | \n",
+ " 36.072968 | \n",
+ " 2024-03-13 15:59:51.012413979-04:00 | \n",
+ " 37.3871 | \n",
+ "
\n",
+ " \n",
+ " | 2024-03-13 15:59:51-04:00 | \n",
+ " 36.0800 | \n",
+ " 36.100 | \n",
+ " 36.0650 | \n",
+ " 36.070 | \n",
+ " 321567 | \n",
+ " 36.074857 | \n",
+ " 2024-03-13 15:59:59.987827063-04:00 | \n",
+ " 37.3871 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
2934 rows × 8 columns
\n",
+ "
testlist_id |
Running |
RunnerId |
+ Market |
diff --git a/v2realbot/static/js/tables/runmanager/init.js b/v2realbot/static/js/tables/runmanager/init.js
index 2d8e358..2b72bfe 100644
--- a/v2realbot/static/js/tables/runmanager/init.js
+++ b/v2realbot/static/js/tables/runmanager/init.js
@@ -45,7 +45,8 @@ function initialize_runmanagerRecords() {
{data: 'valid_to', visible: true},
{data: 'testlist_id', visible: true},
{data: 'strat_running', visible: true},
- {data: 'runner_id', visible: true},
+ {data: 'runner_id', visible: true},
+ {data: 'market', visible: true},
],
paging: true,
processing: true,
diff --git a/v2realbot/tools/loadbatch.py b/v2realbot/tools/loadbatch.py
new file mode 100644
index 0000000..7f7257f
--- /dev/null
+++ b/v2realbot/tools/loadbatch.py
@@ -0,0 +1,316 @@
+import matplotlib
+import matplotlib.dates as mdates
+#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg'
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+from datetime import datetime
+from typing import List
+from enum import Enum
+import numpy as np
+import v2realbot.controller.services as cs
+from rich import print
+from v2realbot.common.model import AnalyzerInputs
+from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType
+from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print
+from pathlib import Path
+from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY
+from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide
+from io import BytesIO
+from v2realbot.utils.historicals import get_historical_bars
+from alpaca.data.timeframe import TimeFrame, TimeFrameUnit
+from collections import defaultdict
+from scipy.stats import zscore
+from io import BytesIO
+from typing import Tuple, Optional, List
+from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType
+from collections import Counter
+import vectorbtpro as vbt
+
+ # Function to add 23 seconds to the last datetime (if it exists and is the same day)
+def adjust_datetime_iteratively(df, resolution):
+ adjusted_times = []
+ for i, current_time in enumerate(df.index):
+ if i == 0:
+ # The first entry is unchanged
+ adjusted_times.append(current_time)
+ continue
+
+ previous_time = adjusted_times[-1]
+ # Check if it's the same day
+ if previous_time.date() == current_time.date():
+ # Add resolution to the previous datetime
+ adjusted_time = previous_time + pd.Timedelta(seconds=resolution)
+ else:
+ # Different day, leave it as is
+ adjusted_time = current_time
+
+ adjusted_times.append(adjusted_time)
+
+ # Update DataFrame index
+ df.index = pd.DatetimeIndex(adjusted_times)
+ return df
+
+def convert_to_dataframe(ohlcv):
+ """
+ Convert a dictionary containing OHLCV data into a pandas DataFrame.
+
+ Parameters:
+ ohlcv (dict): Dictionary containing OHLCV data.
+ It should have keys 'time', 'open', 'high', 'low', 'close', 'volume', 'updated'.
+ 'time' should be a list of float timestamps.
+ 'updated' should be a list of Python datetimes in UTC time zone.
+
+ Returns:
+ pd.DataFrame: DataFrame containing the OHLCV data with the index converted to East coast US time.
+ """
+
+ #pokud existuje key index, tak menime na custom_index, aby nedelal neplechu v pd
+ try:
+ if ohlcv.get('index', False):
+ ohlcv['custom_index'] = ohlcv.pop('index')
+ except Exception as e:
+ pass
+
+ #keys that should not go uppercase letter first
+ keys_not_to_upper = ["time", "updated"]
+
+ # Update keys not in the exclusion list
+ for key in list(ohlcv.keys()): # Iterate over a copy of the keys
+ if key not in keys_not_to_upper:
+ ohlcv[key.title()] = ohlcv.pop(key)
+
+ # Create DataFrame from the dictionary
+ df = pd.DataFrame(ohlcv)
+
+ # Convert 'time' to datetime and set as index
+ df['time'] = pd.to_datetime(df['time'], unit='s', utc=True)
+ df.set_index('time', inplace=True)
+ # Convert index to East coast US time zone
+ df.index = df.index.tz_convert('US/Eastern')
+ if 'updated' in df.columns:
+ df['updated'] = pd.to_datetime(df['updated'], unit='s', utc=True)
+ df['updated'] = df['updated'].dt.tz_convert('US/Eastern')
+
+ return df
+
+def load_batch(runner_ids: List = None, batch_id: str = None, space_resolution_evenly = False, main_session_only = True, merge_ind2bars = True, bars_columns = ['Open', 'High', 'Low', 'Close', 'Volume', 'Vwap'], indicators_columns = []) -> Tuple[int, dict]:
+ """Load batches (all runners from single batch) into pandas dataframes
+
+ Args:
+ runner_ids (List, optional): A list of runner identifiers (e.g., stock tickers). Defaults to None.
+ batch_id (str, optional): The ID of a specific batch to retrieve. Defaults to None.
+ merge_ind2bars (bool, optional): merge indicator into bars dataframe. Defaults to True.
+ bars_columns (list, optional): List of columns to keep in bars df. Defaults to ['Open', 'High', 'Low', 'Close', 'Volume', 'Vwap'].
+ indicators_columns (list, optional): List of columns to keep in indicators df. Defaults to an empty list.
+ space_resoution_evenly: If True then, it alters index so it is spaced evenly in given resolution in ['resooution']
+ Returns:
+ Tuple[int, dict]: A tuple containing:
+ * An integer potentially representing a status code or data count.
+ * A dictionary with keys bars, indicators and cbar_indicators - with pandas dataframe
+ """
+
+ if runner_ids is None and batch_id is None:
+ return -2, f"runner_id or batch_id must be present", 0
+
+ if batch_id is not None:
+ res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id)
+
+ if res != 0:
+ print(f"no batch {batch_id} found")
+ return -1, f"no batch {batch_id} found", 0
+
+ #DATA PREPARATION
+ bars = None
+ indicators = None
+ cnt = 0
+ dfs = dict(bars=[], indicators=[],cbar_indicators=[])
+ resolution = None
+ for id in runner_ids:
+ cnt += 1
+ #get runner detail
+ res, sada =cs.get_archived_runner_details_byID(id)
+ if res != 0:
+ print(f"no runner {id} found")
+ return -1, f"no runner {id} found", 0
+
+ if resolution is None:
+ resolution = sada["bars"]["resolution"][0]
+ print(f"Resolution : {resolution}")
+
+ #add daily bars limited to required columns, we keep updated as its mapping column to indicators
+ bars = convert_to_dataframe(sada["bars"])[bars_columns + ["updated"]]
+ #bars = bars.loc[:, bars_columns]
+
+ indicators = convert_to_dataframe(sada["indicators"][0])[indicators_columns]
+
+ #join indicators to bars dataframe
+ if merge_ind2bars:
+ #merge, time v indicators odpovida udpated v bars
+ bars = bars.reset_index()
+ bars = pd.merge(bars, indicators, left_on="updated", right_on="time", how="left")
+ bars = bars.set_index("time")
+ else:
+ dfs["indicators"].append(indicators)
+
+ #drop updated as mapping column
+ #bars = bars.drop("updated", axis=1)
+ dfs["bars"].append(bars)
+
+ #indicators = sada["indicators"][0]
+ #cbar_indicators = sada["indicators"][1]
+ #merge all days into single df
+ for key in dfs:
+ if len(dfs[key])>0:
+ concat_df = pd.concat(dfs[key], axis=0)
+ concat_df = concat_df.between_time('9:30', '16:00') if main_session_only else concat_df
+
+ # Count the number of duplicates (excluding the first occurrence)
+ num_duplicates = concat_df.index.duplicated().sum()
+
+ if num_duplicates > 0:
+ print(f"NOTE: DUPLICATES {num_duplicates}/{len(concat_df)} in {key}. REMOVING.")
+ concat_df = concat_df[~concat_df.index.duplicated()]
+
+ num_duplicates = concat_df.index.duplicated().sum()
+ print(f"Now there are {num_duplicates}/{len(concat_df)}")
+
+ if space_resolution_evenly and key != "cbar_indicators":
+ # Apply rounding to the datetime index according to resolution (in seconds)
+ concat_df = adjust_datetime_iteratively(concat_df, resolution)
+
+ dfs[key] = concat_df
+ return 0, dfs
+
+if __name__ == "__main__":
+ res, df = load_batch(batch_id="e44a5075", space_resolution_evenly=True, indicators_columns=["Rsi14"], main_session_only=False)
+ if res < 0:
+ print("Error" + str(res) + str(df))
+ print(df)
+ df = df["bars"]
+ print(df.info(), df.head())
+ #filter columns
+ #columns_to_keep = ['Open', 'High', 'Low', 'Close', 'Volume', 'Vwap']
+ #df = df.loc[:, columns_to_keep]
+ #df = df.rename(columns={'index': 'custom_index'})
+ print(df.info(), df.head(), df.describe())
+ #filter times
+ #df = df.between_time('9:30', '16:00')
+ print(df.info())
+ # Set the frequency to 23 seconds
+ #df.index.freq = pd.tseries.offsets.Second(23)
+ # Check the frequency of the index
+
+ # Resample and aggregate the data
+ # resampled_df = df.resample('23S').agg({
+ # 'open': 'first',
+ # 'high': 'max',
+ # 'low': 'min',
+ # 'close': 'last',
+ # 'volume': 'sum'
+ # })
+
+ #df.index.freq = pd.infer_freq(df.index)
+ #print(df.index.freq)
+
+
+ # Set the frequency of the index explicitly - if it exists like 1T etc, if doesnt exists then custom_frequency will be used
+ #df.index.freq = pd.date_range(start=df.index[0], periods=len(df), freq='23S')
+
+ print(df.info())
+
+ vbt.settings.set_theme("dark")
+ vbt.settings['plotting']['layout']['width'] = 1280
+ vbt.settings.plotting.auto_rangebreaks = True
+
+ #naloadujeme do vbt symbol as column
+ bar_data = vbt.Data.from_data({"BAC": df}, tz_convert="US/Eastern")
+ print(bar_data)
+ print(bar_data.close)
+
+ print(bar_data.data["BAC"]["Rsi14"])
+ bar_data.data["BAC"]["Rsi14"].vbt.plot().show()
+ print(bar_data["Rsi14"])
+
+
+ #ohlcv plot (sublot 2x1)
+ bar_data.data["BAC"].vbt.ohlcv.plot().show()
+
+ #create two subplots 3x1 (ohlcv + RSI)
+ # fig = vbt.make_subplots(rows=3, cols=1)
+ # bar_data.data["BAC"].vbt.ohlcv.plot(add_trace_kwargs=dict(row=1, col=1),fig=fig)
+ # bar_data.data["BAC"]["Rsi14"].vbt.plot(add_trace_kwargs=dict(row=3, col=1),fig=fig)
+ # fig.show()
+
+ #create subplots with alternate Y axis - RSI overlay
+ fig1 = vbt.make_subplots(specs=[[{"secondary_y": True}]])
+ bar_data.data["BAC"]["Close"].vbt.plot(add_trace_kwargs=dict(secondary_y=False),fig=fig1)
+ bar_data.data["BAC"].vbt.plot(add_trace_kwargs=dict(secondary_y=True),fig=fig1)
+ fig1.show()
+
+ puv_df = bar_data.data["BAC"]
+
+ bar_data23s = bar_data[["Open", "High", "Low", "Close", "Volume"]]
+ print(bar_data23s)
+ #resample by vbt
+ bar_data46s = bar_data23s.get().resample("46s").agg({
+ "Open": "first",
+ "High": "max",
+ "Low": "min",
+ "Close": "last",
+ "Volume": "sum"
+ })
+
+ print(bar_data46s)
+ res_data = bar_data46s.data["BAC"]
+ #bar_data23s.data["BAC"].ptable()
+ #bar_data23s = bar_data.resample("23S")
+ print(bar_data46s)
+ print(bar_data46s.close)
+ vbt.settings.plotting.auto_rangebreaks = True
+ bar_data46s.data["BAC"].vbt.ohlcv.plot().show()
+
+ #TARGET DAYS - only one day or range
+ # Target Date
+ #target_date = pd.to_datetime('2023-10-12', tz='US/Eastern')
+
+ # Date Range
+ start_date = pd.to_datetime('2024-03-12')
+ #end_date = pd.to_datetime('2023-10-14')
+
+ new_data = bar_data.transform(lambda df: df[df.index.date == start_date.date()])
+ #range filtered_data = data[(data.index >= start_date) & (data.index <= end_date)
+
+ print(new_data)
+ new_data.data["BAC"].vbt.ohlcv.plot().show()
+
+
+ # Filtering RANGE or DAY
+ # filtered_data = data[(data.index >= start_date) & (data.index <= end_date)]g
+ # filtered_data = data[data.index.date == target_date.date()]
+
+
+
+ #custom aggregagation
+ # ohlcv_agg = pd.DataFrame({
+ # 'Open': df.resample('1T')['Open'].first(),
+ # 'High': df.resample('1T')['High'].max(),
+ # 'Low': df.resample('1T')['Low'].min(),
+ # 'Close': df.resample('1T')['Close'].last(),
+ # 'Volume': df.resample('1T')['Volume'].sum()
+ # })
+
+ #Define a custom frequency with a timedelta of 23 seconds
+ # custom_frequency = pd.tseries.offsets.DateOffset(seconds=23)
+
+ # # Create a new DataFrame with the desired frequency
+ # new_index = pd.date_range(start=df.index[0], end=df.index[-1], freq=custom_frequency)
+ # new_df = pd.DataFrame(index=new_index)
+
+ # # Reindex the DataFrame
+ # df = df.reindex(new_df.index)
+
+ # # Now you can check the frequency of the index
+ # print(df.index.freq)
+
+