diff --git a/v2realbot/controller/services.py b/v2realbot/controller/services.py index fac1ec8..f6de492 100644 --- a/v2realbot/controller/services.py +++ b/v2realbot/controller/services.py @@ -37,7 +37,7 @@ from v2realbot.strategyblocks.indicators.indicators_hub import populate_dynamic_ from v2realbot.interfaces.backtest_interface import BacktestInterface import os from v2realbot.reporting.metricstoolsimage import generate_trading_report_image -import gc +#import gc #from pyinstrument import Profiler #adding lock to ensure thread safety of TinyDB (in future will be migrated to proper db) lock = Lock() @@ -524,7 +524,7 @@ def batch_run_manager(id: UUID, runReq: RunRequest, rundays: list[RunDay]): except Exception as e: print("Nepodarilo se vytvorit report image", str(e)+format_exc()) - gc.collect() + #gc.collect() #stratin run def run_stratin(id: UUID, runReq: RunRequest, synchronous: bool = False, inter_batch_params: dict = None): diff --git a/v2realbot/loader/trade_offline_streamer.py b/v2realbot/loader/trade_offline_streamer.py index ad68051..d57fd5f 100644 --- a/v2realbot/loader/trade_offline_streamer.py +++ b/v2realbot/loader/trade_offline_streamer.py @@ -22,7 +22,8 @@ from rich import print import queue from alpaca.trading.models import Calendar from tqdm import tqdm - +import time +from traceback import format_exc """ Trade offline data streamer, based on Alpaca historical data. """ @@ -101,6 +102,19 @@ class Trade_Offline_Streamer(Thread): #REFACTOR STARTS HERE #print(f"{self.time_from=} {self.time_to=}") + def get_calendar_with_retry(self, calendar_request, max_retries=3, delay=4): + attempts = 0 + while attempts < max_retries: + try: + cal_dates = self.clientTrading.get_calendar(calendar_request) + return cal_dates + except Exception as e: + attempts += 1 + if attempts >= max_retries: + raise + print(f"Attempt {attempts}: Error occurred - {e}. Retrying in {delay} seconds...") + time.sleep(delay) + if OFFLINE_MODE: #just one day - same like time_from den = str(self.time_to.date()) @@ -108,8 +122,15 @@ class Trade_Offline_Streamer(Thread): cal_dates = [bt_day] else: calendar_request = GetCalendarRequest(start=self.time_from,end=self.time_to) - cal_dates = self.clientTrading.get_calendar(calendar_request) - #ic(cal_dates) + + #toto zatim workaround - dat do retry funkce a obecne vymyslet exception handling, abych byl notifikovan a bylo videt okamzite v logu a na frontendu + try: + cal_dates = self.clientTrading.get_calendar(calendar_request) + except Exception as e: + print("CHYBA - retrying in 4s: " + str(e) + format_exc()) + time.sleep(5) + cal_dates = self.clientTrading.get_calendar(calendar_request) + #zatim podpora pouze main session #zatim podpora pouze 1 symbolu, predelat na froloop vsech symbolu ze symbpole diff --git a/v2realbot/main.py b/v2realbot/main.py index 0e673a5..1faca47 100644 --- a/v2realbot/main.py +++ b/v2realbot/main.py @@ -33,8 +33,9 @@ from time import sleep import v2realbot.reporting.metricstools as mt from v2realbot.reporting.metricstoolsimage import generate_trading_report_image from traceback import format_exc +from v2realbot.reporting.optimizecutoffs import find_optimal_cutoff #from async io import Queue, QueueEmpty - +# # install() # ic.configureOutput(includeContext=True) # def threadName(): @@ -561,6 +562,20 @@ def _generate_report_image(runner_ids: list[UUID]): except Exception as e: raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {str(e)}" + format_exc()) + +#TODO toto bude zaklad pro obecnou funkci, ktera bude volat ruzne analyzy +#vstupem bude obecny objekt, ktery ponese nazev analyzy + atributy +@app.post("/batches/optimizecutoff/{batch_id}", dependencies=[Depends(api_key_auth)], responses={200: {"content": {"image/png": {}}}}) +def _generate_analysis(batch_id: str): + try: + res, stream = find_optimal_cutoff(batch_id=batch_id, steps=50, stream=True) + if res == 0: return StreamingResponse(stream, media_type="image/png",headers={"Content-Disposition": "attachment; filename=optimizedcutoff.png"}) + elif res < 0: + raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {res}:{id}") + except Exception as e: + raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {str(e)}" + format_exc()) + + #TestList APIS - do budoucna predelat SQL do separatnich funkci @app.post('/testlists/', dependencies=[Depends(api_key_auth)]) def create_record(testlist: TestList): diff --git a/v2realbot/reporting/archive/optimizecutoff.py b/v2realbot/reporting/archive/optimizecutoff.py new file mode 100644 index 0000000..2b36e60 --- /dev/null +++ b/v2realbot/reporting/archive/optimizecutoff.py @@ -0,0 +1,166 @@ +import matplotlib +import matplotlib.dates as mdates +#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg' +import matplotlib.pyplot as plt +import seaborn as sns +import pandas as pd +from datetime import datetime +from typing import List +from enum import Enum +import numpy as np +import v2realbot.controller.services as cs +from rich import print +from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType +from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print +from pathlib import Path +from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY +from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide +from io import BytesIO +from v2realbot.utils.historicals import get_historical_bars +from alpaca.data.timeframe import TimeFrame, TimeFrameUnit +from collections import defaultdict +# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere + + +#TODO: +#OPTIMALIZOVAT TENTO nebo DRUHY soubor +#Vyzkouset v realu +#ZKUSIT NA RELATIVNI PROFIT TAKE + +#ZAPRACOVAT DO GRAFU nebo do nejakeho toolu + + + +def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False): + + #TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se + #TODO list of runner_ids + #TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera + + if runner_ids is None and batch_id is None: + return -2, f"runner_id or batch_id must be present" + + if batch_id is not None: + res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id) + + if res != 0: + print(f"no batch {batch_id} found") + return -1, f"no batch {batch_id} found" + + trades = [] + cnt_max = len(runner_ids) + cnt = 0 + #zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch + end_date = None + start_date = None + for id in runner_ids: + cnt += 1 + #get runner + res, sada =cs.get_archived_runner_header_byID(id) + if res != 0: + print(f"no runner {id} found") + return -1, f"no runner {id} found" + + print("archrunner") + #print(sada) + + if cnt == 1: + start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started + if cnt == cnt_max: + end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped + # Parse trades + + trades_dicts = sada.metrics["prescr_trades"] + + for trade_dict in trades_dicts: + trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None + trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None + trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None + trades.append(Trade(**trade_dict)) + + #print(trades) + + # symbol = sada.symbol + # #hour bars for backtested period + # print(start_date,end_date) + # bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour) + # print("bars for given period",bars) + # """Bars a dictionary with the following keys: + # * high: A list of high prices + # * low: A list of low prices + # * volume: A list of volumes + # * close: A list of close prices + # * hlcc4: A list of HLCC4 indicators + # * open: A list of open prices + # * time: A list of times in UTC (ISO 8601 format) + # * trades: A list of number of trades + # * resolution: A list of resolutions (all set to 'D') + # * confirmed: A list of booleans (all set to True) + # * vwap: A list of VWAP indicator + # * updated: A list of booleans (all set to True) + # * index: A list of integers (from 0 to the length of the list of daily bars) + # """ + + # Filter to only use trades with status 'CLOSED' + closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED] + + print(closed_trades) + + if len(closed_trades) == 0: + return -1, "image generation no closed trades" + + # Group trades by date and calculate daily profits + trades_by_day = defaultdict(list) + for trade in trades: + if trade.status == TradeStatus.CLOSED and trade.exit_time: + trade_day = trade.exit_time.date() + trades_by_day[trade_day].append(trade) + + # Define a range of potential cutoff values + # min_profit = min(trade.profit for trade in trades if trade.profit is not None) + # max_profit = max(trade.profit for trade in trades if trade.profit is not None) + min_profit = 0 + max_profit = 1000 + potential_cutoffs = np.linspace(min_profit, max_profit, 100) + + # Function to calculate total profit for a given cutoff + def calculate_total_profit(cutoff): + total_profit = 0 + for day, day_trades in trades_by_day.items(): + daily_profit = 0 + for trade in day_trades: + daily_profit += trade.profit + if daily_profit >= cutoff: # Stop if daily profit reaches the cutoff + break + total_profit += min(daily_profit, cutoff) # Add the minimum of daily profit or cutoff + return total_profit + + # Evaluate each cutoff + cutoff_profits = {cutoff: calculate_total_profit(cutoff) for cutoff in potential_cutoffs} + + # Find the optimal cutoff + optimal_cutoff = max(cutoff_profits, key=cutoff_profits.get) + max_profit = cutoff_profits[optimal_cutoff] + + # Plotting + plt.figure(figsize=(10, 6)) + plt.plot(list(cutoff_profits.keys()), list(cutoff_profits.values()), label='Total Profit') + plt.scatter(optimal_cutoff, max_profit, color='red', label='Optimal Cutoff') + plt.title('Optimal Intra-Day Profit Cutoff Analysis') + plt.xlabel('Profit Cutoff') + plt.ylabel('Total Profit') + plt.legend() + plt.grid(True) + plt.savefig('optimal_cutoff.png') + + return optimal_cutoff, max_profit + + +# Example usage +# trades = [list of Trade objects] +if __name__ == '__main__': + # id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"] + # generate_trading_report_image(runner_ids=id_list) + batch_id = "90973e57" + optimal_cutoff, max_profit = find_optimal_cutoff(batch_id=batch_id) + print(f"Optimal Cutoff: {optimal_cutoff}, Max Profit: {max_profit}") \ No newline at end of file diff --git a/v2realbot/reporting/archive/optimizecutoffprofloss.py b/v2realbot/reporting/archive/optimizecutoffprofloss.py new file mode 100644 index 0000000..f47d6b6 --- /dev/null +++ b/v2realbot/reporting/archive/optimizecutoffprofloss.py @@ -0,0 +1,167 @@ +import matplotlib +import matplotlib.dates as mdates +#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg' +import matplotlib.pyplot as plt +import seaborn as sns +import pandas as pd +from datetime import datetime +from typing import List +from enum import Enum +import numpy as np +import v2realbot.controller.services as cs +from rich import print +from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType +from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print +from pathlib import Path +from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY +from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide +from io import BytesIO +from v2realbot.utils.historicals import get_historical_bars +from alpaca.data.timeframe import TimeFrame, TimeFrameUnit +from collections import defaultdict +# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere + +#LOSS and PROFIT without GRAPH +def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False): + + #TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se + #TODO list of runner_ids + #TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera + + if runner_ids is None and batch_id is None: + return -2, f"runner_id or batch_id must be present" + + if batch_id is not None: + res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id) + + if res != 0: + print(f"no batch {batch_id} found") + return -1, f"no batch {batch_id} found" + + trades = [] + cnt_max = len(runner_ids) + cnt = 0 + #zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch + end_date = None + start_date = None + for id in runner_ids: + cnt += 1 + #get runner + res, sada =cs.get_archived_runner_header_byID(id) + if res != 0: + print(f"no runner {id} found") + return -1, f"no runner {id} found" + + print("archrunner") + #print(sada) + + if cnt == 1: + start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started + if cnt == cnt_max: + end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped + # Parse trades + + trades_dicts = sada.metrics["prescr_trades"] + + for trade_dict in trades_dicts: + trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None + trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None + trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None + trades.append(Trade(**trade_dict)) + + #print(trades) + + # symbol = sada.symbol + # #hour bars for backtested period + # print(start_date,end_date) + # bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour) + # print("bars for given period",bars) + # """Bars a dictionary with the following keys: + # * high: A list of high prices + # * low: A list of low prices + # * volume: A list of volumes + # * close: A list of close prices + # * hlcc4: A list of HLCC4 indicators + # * open: A list of open prices + # * time: A list of times in UTC (ISO 8601 format) + # * trades: A list of number of trades + # * resolution: A list of resolutions (all set to 'D') + # * confirmed: A list of booleans (all set to True) + # * vwap: A list of VWAP indicator + # * updated: A list of booleans (all set to True) + # * index: A list of integers (from 0 to the length of the list of daily bars) + # """ + + # Filter to only use trades with status 'CLOSED' + closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED] + + #print(closed_trades) + + if len(closed_trades) == 0: + return -1, "image generation no closed trades" + + # Group trades by date and calculate daily profits + trades_by_day = defaultdict(list) + for trade in trades: + if trade.status == TradeStatus.CLOSED and trade.exit_time: + trade_day = trade.exit_time.date() + trades_by_day[trade_day].append(trade) + + # Define ranges for loss and profit cutoffs + min_profit = 0 + max_profit = 1000 + profit_cutoffs = np.linspace(min_profit, max_profit, 20) + + min_loss = 0 + max_loss = -1000 + loss_cutoffs = np.linspace(min_loss, max_loss, 20) + #print(profit_cutoffs, loss_cutoffs) + # Function to calculate total profit for a given cutoff + def calculate_total_profit(profit_cutoff, loss_cutoff): + total_profit = 0 + for day, day_trades in trades_by_day.items(): + daily_profit = 0 + for trade in day_trades: + daily_profit += trade.profit + if daily_profit >= profit_cutoff or daily_profit <= loss_cutoff: + break + total_profit += daily_profit + return total_profit + + # Initialize variables to store optimal cutoffs and max total profit + optimal_profit_cutoff = max_profit + optimal_loss_cutoff = min_loss + max_total_profit = float('-inf') + + # Initialize a matrix to store total profits for each combination of cutoffs + total_profits_matrix = np.zeros((len(profit_cutoffs), len(loss_cutoffs))) + + for i, profit_cutoff in enumerate(profit_cutoffs): + for j, loss_cutoff in enumerate(loss_cutoffs): + total_profit = calculate_total_profit(profit_cutoff, loss_cutoff) + total_profits_matrix[i, j] = total_profit + if total_profit > max_total_profit: + max_total_profit = total_profit + optimal_profit_cutoff = profit_cutoff + optimal_loss_cutoff = loss_cutoff + + # Plotting the results as a heatmap + plt.figure(figsize=(20, 15)) + sns.heatmap(total_profits_matrix, xticklabels=np.round(loss_cutoffs, 2), yticklabels=np.round(profit_cutoffs, 2), + annot=True, fmt=".0f", cmap="viridis") #fmt=".0f", + #plasma, coolmap, inferno, viridis + plt.title("Total Profit for Combinations of Profit and Loss Cutoffs") + plt.xlabel("Loss Cutoff") + plt.ylabel("Profit Cutoff") + plt.savefig('optimal_cutoff.png') + + return optimal_profit_cutoff, optimal_loss_cutoff, max_total_profit + +# Example usage +# trades = [list of Trade objects] +if __name__ == '__main__': + # id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"] + # generate_trading_report_image(runner_ids=id_list) + batch_id = "c76b4414" + optimal_profit_cutoff, optimal_loss_cutoff, max_profit = find_optimal_cutoff(batch_id=batch_id) + print(f"Optimal Profit Cutoff: {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_profit}") \ No newline at end of file diff --git a/v2realbot/reporting/archive/optimizecutoffv2.py b/v2realbot/reporting/archive/optimizecutoffv2.py new file mode 100644 index 0000000..a92fe73 --- /dev/null +++ b/v2realbot/reporting/archive/optimizecutoffv2.py @@ -0,0 +1,169 @@ +import matplotlib +import matplotlib.dates as mdates +#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg' +import matplotlib.pyplot as plt +import seaborn as sns +import pandas as pd +from datetime import datetime +from typing import List +from enum import Enum +import numpy as np +import v2realbot.controller.services as cs +from rich import print +from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType +from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print +from pathlib import Path +from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY +from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide +from io import BytesIO +from v2realbot.utils.historicals import get_historical_bars +from alpaca.data.timeframe import TimeFrame, TimeFrameUnit +from collections import defaultdict +# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere + +#LOSS and PROFIT without GRAPH +def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False): + + #TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se + #TODO list of runner_ids + #TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera + + if runner_ids is None and batch_id is None: + return -2, f"runner_id or batch_id must be present" + + if batch_id is not None: + res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id) + + if res != 0: + print(f"no batch {batch_id} found") + return -1, f"no batch {batch_id} found" + + trades = [] + cnt_max = len(runner_ids) + cnt = 0 + #zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch + end_date = None + start_date = None + for id in runner_ids: + cnt += 1 + #get runner + res, sada =cs.get_archived_runner_header_byID(id) + if res != 0: + print(f"no runner {id} found") + return -1, f"no runner {id} found" + + print("archrunner") + #print(sada) + + if cnt == 1: + start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started + if cnt == cnt_max: + end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped + # Parse trades + + trades_dicts = sada.metrics["prescr_trades"] + + for trade_dict in trades_dicts: + trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None + trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None + trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None + trades.append(Trade(**trade_dict)) + + #print(trades) + + # symbol = sada.symbol + # #hour bars for backtested period + # print(start_date,end_date) + # bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour) + # print("bars for given period",bars) + # """Bars a dictionary with the following keys: + # * high: A list of high prices + # * low: A list of low prices + # * volume: A list of volumes + # * close: A list of close prices + # * hlcc4: A list of HLCC4 indicators + # * open: A list of open prices + # * time: A list of times in UTC (ISO 8601 format) + # * trades: A list of number of trades + # * resolution: A list of resolutions (all set to 'D') + # * confirmed: A list of booleans (all set to True) + # * vwap: A list of VWAP indicator + # * updated: A list of booleans (all set to True) + # * index: A list of integers (from 0 to the length of the list of daily bars) + # """ + + # Filter to only use trades with status 'CLOSED' + closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED] + + print(closed_trades) + + if len(closed_trades) == 0: + return -1, "image generation no closed trades" + + # Group trades by date and calculate daily profits + trades_by_day = defaultdict(list) + for trade in trades: + if trade.status == TradeStatus.CLOSED and trade.exit_time: + trade_day = trade.exit_time.date() + trades_by_day[trade_day].append(trade) + + # Define ranges for loss and profit cutoffs + min_profit = 50 + max_profit = 700 # Set an upper bound based on your data + profit_cutoffs = np.linspace(min_profit, max_profit, 50) # Adjust number of points as needed + + min_loss = -50 + max_loss = -700 # Assuming losses are negative values + loss_cutoffs = np.linspace(min_loss, max_loss, 50) + + def calculate_total_profit(profit_cutoff, loss_cutoff): + total_profit = 0 + for day, day_trades in trades_by_day.items(): + daily_profit = 0 + for trade in day_trades: + daily_profit += trade.profit + if daily_profit >= profit_cutoff or daily_profit <= loss_cutoff: + break + total_profit += daily_profit + return total_profit + + # Evaluate each combination of cutoffs + optimal_profit_cutoff = max_profit + optimal_loss_cutoff = min_loss + max_total_profit = float('-inf') + + for profit_cutoff in profit_cutoffs: + for loss_cutoff in loss_cutoffs: + total_profit = calculate_total_profit(profit_cutoff, loss_cutoff) + if total_profit > max_total_profit: + max_total_profit = total_profit + optimal_profit_cutoff = profit_cutoff + optimal_loss_cutoff = loss_cutoff + + print(f"Optimal Profit Cutoff: {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_total_profit}") + + # Optional: Plot the results or return them for further analysis + + return optimal_profit_cutoff, optimal_loss_cutoff, max_total_profit + + # # Plotting + # plt.figure(figsize=(10, 6)) + # plt.plot(list(cutoff_profits.keys()), list(cutoff_profits.values()), label='Total Profit') + # plt.scatter(optimal_cutoff, max_profit, color='red', label='Optimal Cutoff') + # plt.title('Optimal Intra-Day Profit Cutoff Analysis') + # plt.xlabel('Profit Cutoff') + # plt.ylabel('Total Profit') + # plt.legend() + # plt.grid(True) + # plt.savefig('optimal_cutoff.png') + + # return optimal_cutoff, max_profit + +# Example usage +# trades = [list of Trade objects] +if __name__ == '__main__': + # id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"] + # generate_trading_report_image(runner_ids=id_list) + batch_id = "c76b4414" + optimal_profit_cutoff, optimal_loss_cutoff, max_profit = find_optimal_cutoff(batch_id=batch_id) + print(f"Optimal Profit Cutoff: {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_profit}") \ No newline at end of file diff --git a/v2realbot/reporting/optimizecutoffs.py b/v2realbot/reporting/optimizecutoffs.py new file mode 100644 index 0000000..42740bf --- /dev/null +++ b/v2realbot/reporting/optimizecutoffs.py @@ -0,0 +1,241 @@ +import matplotlib +import matplotlib.dates as mdates +#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg' +import matplotlib.pyplot as plt +import seaborn as sns +import pandas as pd +from datetime import datetime +from typing import List +from enum import Enum +import numpy as np +import v2realbot.controller.services as cs +from rich import print +from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType +from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print +from pathlib import Path +from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY +from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide +from io import BytesIO +from v2realbot.utils.historicals import get_historical_bars +from alpaca.data.timeframe import TimeFrame, TimeFrameUnit +from collections import defaultdict +from scipy.stats import zscore +from io import BytesIO +# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere + +#LOSS and PROFIT without GRAPH +def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False, rem_outliers:bool = False, file: str = "optimalcutoff.png",steps:int = 50): + + #TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se + #TODO list of runner_ids + #TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera + + if runner_ids is None and batch_id is None: + return -2, f"runner_id or batch_id must be present" + + if batch_id is not None: + res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id) + + if res != 0: + print(f"no batch {batch_id} found") + return -1, f"no batch {batch_id} found" + + trades = [] + cnt_max = len(runner_ids) + cnt = 0 + #zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch + end_date = None + start_date = None + for id in runner_ids: + cnt += 1 + #get runner + res, sada =cs.get_archived_runner_header_byID(id) + if res != 0: + print(f"no runner {id} found") + return -1, f"no runner {id} found" + + #print("archrunner") + #print(sada) + + if cnt == 1: + start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started + if cnt == cnt_max: + end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped + # Parse trades + + trades_dicts = sada.metrics["prescr_trades"] + + for trade_dict in trades_dicts: + trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None + trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None + trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None + trades.append(Trade(**trade_dict)) + + #print(trades) + + # symbol = sada.symbol + # #hour bars for backtested period + # print(start_date,end_date) + # bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour) + # print("bars for given period",bars) + # """Bars a dictionary with the following keys: + # * high: A list of high prices + # * low: A list of low prices + # * volume: A list of volumes + # * close: A list of close prices + # * hlcc4: A list of HLCC4 indicators + # * open: A list of open prices + # * time: A list of times in UTC (ISO 8601 format) + # * trades: A list of number of trades + # * resolution: A list of resolutions (all set to 'D') + # * confirmed: A list of booleans (all set to True) + # * vwap: A list of VWAP indicator + # * updated: A list of booleans (all set to True) + # * index: A list of integers (from 0 to the length of the list of daily bars) + # """ + + # Filter to only use trades with status 'CLOSED' + closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED] + + #print(closed_trades) + + if len(closed_trades) == 0: + return -1, "image generation no closed trades" + + # # Group trades by date and calculate daily profits + # trades_by_day = defaultdict(list) + # for trade in trades: + # if trade.status == TradeStatus.CLOSED and trade.exit_time: + # trade_day = trade.exit_time.date() + # trades_by_day[trade_day].append(trade) + + # Precompute daily cumulative profits + daily_cumulative_profits = defaultdict(list) + for trade in trades: + if trade.status == TradeStatus.CLOSED and trade.exit_time: + day = trade.exit_time.date() + daily_cumulative_profits[day].append(trade.profit) + + for day in daily_cumulative_profits: + daily_cumulative_profits[day] = np.cumsum(daily_cumulative_profits[day]) + + + if rem_outliers: + # Remove outliers based on z-scores + def remove_outliers(cumulative_profits): + all_profits = [profit[-1] for profit in cumulative_profits.values() if len(profit) > 0] + z_scores = zscore(all_profits) + print(z_scores) + filtered_profits = {} + for day, profits in cumulative_profits.items(): + if len(profits) > 0: + day_z_score = z_scores[list(cumulative_profits.keys()).index(day)] + if abs(day_z_score) < 3: # Adjust threshold as needed + filtered_profits[day] = profits + return filtered_profits + + daily_cumulative_profits = remove_outliers(daily_cumulative_profits) + + + # OPT1 Dynamically calculate profit_range and loss_range - based on eod daily profit + # all_final_profits = [profits[-1] for profits in daily_cumulative_profits.values() if len(profits) > 0] + # max_profit = max(all_final_profits) + # min_profit = min(all_final_profits) + # profit_range = (0, max_profit) if max_profit > 0 else (0, 0) + # loss_range = (min_profit, 0) if min_profit < 0 else (0, 0) + + # OPT2 Calculate profit_range and loss_range based on all cumulative profits + all_cumulative_profits = np.concatenate([profits for profits in daily_cumulative_profits.values()]) + max_cumulative_profit = np.max(all_cumulative_profits) + min_cumulative_profit = np.min(all_cumulative_profits) + profit_range = (0, max_cumulative_profit) if max_cumulative_profit > 0 else (0, 0) + loss_range = (min_cumulative_profit, 0) if min_cumulative_profit < 0 else (0, 0) + + print("Calculated ranges", profit_range, loss_range) + + num_points = steps # Adjust for speed vs accuracy + profit_cutoffs = np.linspace(*profit_range, num_points) + loss_cutoffs = np.linspace(*loss_range, num_points) + + # OPT 3Statically define ranges for loss and profit cutoffs + # profit_range = (0, 1000) # Adjust based on your data + # loss_range = (-1000, 0) + # num_points = 20 # Adjust for speed vs accuracy + + profit_cutoffs = np.linspace(*profit_range, num_points) + loss_cutoffs = np.linspace(*loss_range, num_points) + + total_profits_matrix = np.zeros((len(profit_cutoffs), len(loss_cutoffs))) + + for i, profit_cutoff in enumerate(profit_cutoffs): + for j, loss_cutoff in enumerate(loss_cutoffs): + total_profit = 0 + for daily_profit in daily_cumulative_profits.values(): + cutoff_index = np.where((daily_profit >= profit_cutoff) | (daily_profit <= loss_cutoff))[0] + if cutoff_index.size > 0: + total_profit += daily_profit[cutoff_index[0]] + else: + total_profit += daily_profit[-1] if daily_profit.size > 0 else 0 + total_profits_matrix[i, j] = total_profit + + # Find the optimal combination + optimal_idx = np.unravel_index(total_profits_matrix.argmax(), total_profits_matrix.shape) + optimal_profit_cutoff = profit_cutoffs[optimal_idx[0]] + optimal_loss_cutoff = loss_cutoffs[optimal_idx[1]] + max_profit = total_profits_matrix[optimal_idx] + + # Plotting + # Setting up dark mode for the plots + plt.style.use('dark_background') + + # Optionally, you can further customize colors, labels, and axes + params = { + 'axes.titlesize': 9, + 'axes.labelsize': 8, + 'xtick.labelsize': 9, + 'ytick.labelsize': 9, + 'axes.labelcolor': '#a9a9a9', #a1a3aa', + 'axes.facecolor': '#121722', #'#0e0e0e', #202020', # Dark background for plot area + 'axes.grid': False, # Turn off the grid globally + 'grid.color': 'gray', # If the grid is on, set grid line color + 'grid.linestyle': '--', # Grid line style + 'grid.linewidth': 1, + 'xtick.color': '#a9a9a9', + 'ytick.color': '#a9a9a9', + 'axes.edgecolor': '#a9a9a9' + } + plt.rcParams.update(params) + plt.figure(figsize=(10, 8)) + sns.heatmap(total_profits_matrix, xticklabels=np.rint(loss_cutoffs).astype(int), yticklabels=np.rint(profit_cutoffs).astype(int), cmap="viridis") + plt.xticks(rotation=90) # Rotate x-axis labels to be vertical + plt.yticks(rotation=0) # Keep y-axis labels horizontal + plt.gca().invert_yaxis() + plt.gca().invert_xaxis() + plt.suptitle("Total Profit for Combinations of Profit and Loss Cutoffs", fontsize=16) + plt.title(f"Optimal Profit Cutoff: {optimal_profit_cutoff:.2f}, Optimal Loss Cutoff: {optimal_loss_cutoff:.2f}, Max Profit: {max_profit:.2f}", fontsize=10) + plt.xlabel("Loss Cutoff") + plt.ylabel("Profit Cutoff") + + if stream is False: + plt.savefig(file) + plt.close() + print(f"Optimal Profit Cutoff(rem_outliers:{rem_outliers}): {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_profit}") + return 0, None + else: + # Return the image as a BytesIO stream + img_stream = BytesIO() + plt.savefig(img_stream, format='png') + plt.close() + img_stream.seek(0) # Rewind the stream to the beginning + return 0, img_stream + +# Example usage +# trades = [list of Trade objects] +if __name__ == '__main__': + # id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"] + # generate_trading_report_image(runner_ids=id_list) + batch_id = "c76b4414" + res, val = find_optimal_cutoff(batch_id=batch_id, file="optimal_cutoff_vectorized.png",steps=20) + #res, val = find_optimal_cutoff(batch_id=batch_id, rem_outliers=True, file="optimal_cutoff_vectorized_nooutliers.png") + + print(res,val) \ No newline at end of file diff --git a/v2realbot/static/index.html b/v2realbot/static/index.html index 6932148..83d1433 100644 --- a/v2realbot/static/index.html +++ b/v2realbot/static/index.html @@ -310,6 +310,7 @@ + diff --git a/v2realbot/static/js/archivetables.js b/v2realbot/static/js/archivetables.js index 7ad19bf..5a59e8b 100644 --- a/v2realbot/static/js/archivetables.js +++ b/v2realbot/static/js/archivetables.js @@ -210,6 +210,7 @@ $(document).ready(function () { $('#button_runagain_arch').attr('disabled','disabled'); $('#button_show_arch').attr('disabled','disabled'); $('#button_delete_arch').attr('disabled','disabled'); + $('#button_analyze').attr('disabled','disabled'); $('#button_edit_arch').attr('disabled','disabled'); $('#button_compare_arch').attr('disabled','disabled'); @@ -218,6 +219,7 @@ $(document).ready(function () { if ($(this).hasClass('selected')) { //$(this).removeClass('selected'); $('#button_show_arch').attr('disabled','disabled'); + $('#button_analyze').attr('disabled','disabled'); $('#button_runagain_arch').attr('disabled','disabled'); $('#button_delete_arch').attr('disabled','disabled'); $('#button_edit_arch').attr('disabled','disabled'); @@ -225,6 +227,7 @@ $(document).ready(function () { } else { //archiveRecords.$('tr.selected').removeClass('selected'); $(this).addClass('selected'); + $('#button_analyze').attr('disabled',false); $('#button_show_arch').attr('disabled',false); $('#button_runagain_arch').attr('disabled',false); $('#button_delete_arch').attr('disabled',false); @@ -413,12 +416,50 @@ $(document).ready(function () { } }); + //generate batch optimization cutoff (predelat na button pro obecne analyzy batche) + $('#button_analyze').click(function () { + row = archiveRecords.row('.selected').data(); + if (row == undefined || row.batch_id == undefined) { + return + } + console.log(row) + $('#button_analyze').attr('disabled','disabled'); + $.ajax({ + url:"/batches/optimizecutoff/"+row.batch_id, + beforeSend: function (xhr) { + xhr.setRequestHeader('X-API-Key', + API_KEY); }, + method:"POST", + xhrFields: { + responseType: 'blob' + }, + contentType: "application/json", + processData: false, + data: JSON.stringify(row.batch_id), + success:function(blob){ + var url = window.URL || window.webkitURL; + console.log("vraceny obraz", blob) + console.log("url",url.createObjectURL(blob)) + display_image(url.createObjectURL(blob)) + $('#button_analyze').attr('disabled',false); + }, + error: function(xhr, status, error) { + console.log("proc to skace do erroru?") + //window.alert(JSON.stringify(xhr)); + console.log(JSON.stringify(xhr)); + $('#button_analyze').attr('disabled',false); + } + }) + }); + + //generate report button $('#button_report').click(function () { rows = archiveRecords.rows('.selected'); if (rows == undefined) { return } + $('#button_report').attr('disabled','disabled'); runnerIds = [] if(rows.data().length > 0 ) { // Loop through the selected rows and display an alert with each row's ID @@ -444,11 +485,13 @@ $(document).ready(function () { console.log("vraceny obraz", blob) console.log("url",url.createObjectURL(blob)) display_image(url.createObjectURL(blob)) + $('#button_report').attr('disabled',false); }, error: function(xhr, status, error) { console.log("proc to skace do erroru?") //window.alert(JSON.stringify(xhr)); - console.log(JSON.stringify(xhr)); + console.log(JSON.stringify(xhr)); + $('#button_report').attr('disabled',false); } }) });