bugfix + optimizedcutoffs button on gui

This commit is contained in:
David Brazda
2023-11-22 12:18:42 +01:00
parent d621004f9a
commit 65897ce6c4
9 changed files with 830 additions and 7 deletions

View File

@ -37,7 +37,7 @@ from v2realbot.strategyblocks.indicators.indicators_hub import populate_dynamic_
from v2realbot.interfaces.backtest_interface import BacktestInterface
import os
from v2realbot.reporting.metricstoolsimage import generate_trading_report_image
import gc
#import gc
#from pyinstrument import Profiler
#adding lock to ensure thread safety of TinyDB (in future will be migrated to proper db)
lock = Lock()
@ -524,7 +524,7 @@ def batch_run_manager(id: UUID, runReq: RunRequest, rundays: list[RunDay]):
except Exception as e:
print("Nepodarilo se vytvorit report image", str(e)+format_exc())
gc.collect()
#gc.collect()
#stratin run
def run_stratin(id: UUID, runReq: RunRequest, synchronous: bool = False, inter_batch_params: dict = None):

View File

@ -22,7 +22,8 @@ from rich import print
import queue
from alpaca.trading.models import Calendar
from tqdm import tqdm
import time
from traceback import format_exc
"""
Trade offline data streamer, based on Alpaca historical data.
"""
@ -101,6 +102,19 @@ class Trade_Offline_Streamer(Thread):
#REFACTOR STARTS HERE
#print(f"{self.time_from=} {self.time_to=}")
def get_calendar_with_retry(self, calendar_request, max_retries=3, delay=4):
attempts = 0
while attempts < max_retries:
try:
cal_dates = self.clientTrading.get_calendar(calendar_request)
return cal_dates
except Exception as e:
attempts += 1
if attempts >= max_retries:
raise
print(f"Attempt {attempts}: Error occurred - {e}. Retrying in {delay} seconds...")
time.sleep(delay)
if OFFLINE_MODE:
#just one day - same like time_from
den = str(self.time_to.date())
@ -108,8 +122,15 @@ class Trade_Offline_Streamer(Thread):
cal_dates = [bt_day]
else:
calendar_request = GetCalendarRequest(start=self.time_from,end=self.time_to)
#toto zatim workaround - dat do retry funkce a obecne vymyslet exception handling, abych byl notifikovan a bylo videt okamzite v logu a na frontendu
try:
cal_dates = self.clientTrading.get_calendar(calendar_request)
#ic(cal_dates)
except Exception as e:
print("CHYBA - retrying in 4s: " + str(e) + format_exc())
time.sleep(5)
cal_dates = self.clientTrading.get_calendar(calendar_request)
#zatim podpora pouze main session
#zatim podpora pouze 1 symbolu, predelat na froloop vsech symbolu ze symbpole

View File

@ -33,8 +33,9 @@ from time import sleep
import v2realbot.reporting.metricstools as mt
from v2realbot.reporting.metricstoolsimage import generate_trading_report_image
from traceback import format_exc
from v2realbot.reporting.optimizecutoffs import find_optimal_cutoff
#from async io import Queue, QueueEmpty
#
# install()
# ic.configureOutput(includeContext=True)
# def threadName():
@ -561,6 +562,20 @@ def _generate_report_image(runner_ids: list[UUID]):
except Exception as e:
raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {str(e)}" + format_exc())
#TODO toto bude zaklad pro obecnou funkci, ktera bude volat ruzne analyzy
#vstupem bude obecny objekt, ktery ponese nazev analyzy + atributy
@app.post("/batches/optimizecutoff/{batch_id}", dependencies=[Depends(api_key_auth)], responses={200: {"content": {"image/png": {}}}})
def _generate_analysis(batch_id: str):
try:
res, stream = find_optimal_cutoff(batch_id=batch_id, steps=50, stream=True)
if res == 0: return StreamingResponse(stream, media_type="image/png",headers={"Content-Disposition": "attachment; filename=optimizedcutoff.png"})
elif res < 0:
raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {res}:{id}")
except Exception as e:
raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {str(e)}" + format_exc())
#TestList APIS - do budoucna predelat SQL do separatnich funkci
@app.post('/testlists/', dependencies=[Depends(api_key_auth)])
def create_record(testlist: TestList):

View File

@ -0,0 +1,166 @@
import matplotlib
import matplotlib.dates as mdates
#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg'
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from datetime import datetime
from typing import List
from enum import Enum
import numpy as np
import v2realbot.controller.services as cs
from rich import print
from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType
from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print
from pathlib import Path
from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide
from io import BytesIO
from v2realbot.utils.historicals import get_historical_bars
from alpaca.data.timeframe import TimeFrame, TimeFrameUnit
from collections import defaultdict
# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere
#TODO:
#OPTIMALIZOVAT TENTO nebo DRUHY soubor
#Vyzkouset v realu
#ZKUSIT NA RELATIVNI PROFIT TAKE
#ZAPRACOVAT DO GRAFU nebo do nejakeho toolu
def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False):
#TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se
#TODO list of runner_ids
#TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera
if runner_ids is None and batch_id is None:
return -2, f"runner_id or batch_id must be present"
if batch_id is not None:
res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id)
if res != 0:
print(f"no batch {batch_id} found")
return -1, f"no batch {batch_id} found"
trades = []
cnt_max = len(runner_ids)
cnt = 0
#zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch
end_date = None
start_date = None
for id in runner_ids:
cnt += 1
#get runner
res, sada =cs.get_archived_runner_header_byID(id)
if res != 0:
print(f"no runner {id} found")
return -1, f"no runner {id} found"
print("archrunner")
#print(sada)
if cnt == 1:
start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started
if cnt == cnt_max:
end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped
# Parse trades
trades_dicts = sada.metrics["prescr_trades"]
for trade_dict in trades_dicts:
trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None
trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None
trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None
trades.append(Trade(**trade_dict))
#print(trades)
# symbol = sada.symbol
# #hour bars for backtested period
# print(start_date,end_date)
# bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour)
# print("bars for given period",bars)
# """Bars a dictionary with the following keys:
# * high: A list of high prices
# * low: A list of low prices
# * volume: A list of volumes
# * close: A list of close prices
# * hlcc4: A list of HLCC4 indicators
# * open: A list of open prices
# * time: A list of times in UTC (ISO 8601 format)
# * trades: A list of number of trades
# * resolution: A list of resolutions (all set to 'D')
# * confirmed: A list of booleans (all set to True)
# * vwap: A list of VWAP indicator
# * updated: A list of booleans (all set to True)
# * index: A list of integers (from 0 to the length of the list of daily bars)
# """
# Filter to only use trades with status 'CLOSED'
closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED]
print(closed_trades)
if len(closed_trades) == 0:
return -1, "image generation no closed trades"
# Group trades by date and calculate daily profits
trades_by_day = defaultdict(list)
for trade in trades:
if trade.status == TradeStatus.CLOSED and trade.exit_time:
trade_day = trade.exit_time.date()
trades_by_day[trade_day].append(trade)
# Define a range of potential cutoff values
# min_profit = min(trade.profit for trade in trades if trade.profit is not None)
# max_profit = max(trade.profit for trade in trades if trade.profit is not None)
min_profit = 0
max_profit = 1000
potential_cutoffs = np.linspace(min_profit, max_profit, 100)
# Function to calculate total profit for a given cutoff
def calculate_total_profit(cutoff):
total_profit = 0
for day, day_trades in trades_by_day.items():
daily_profit = 0
for trade in day_trades:
daily_profit += trade.profit
if daily_profit >= cutoff: # Stop if daily profit reaches the cutoff
break
total_profit += min(daily_profit, cutoff) # Add the minimum of daily profit or cutoff
return total_profit
# Evaluate each cutoff
cutoff_profits = {cutoff: calculate_total_profit(cutoff) for cutoff in potential_cutoffs}
# Find the optimal cutoff
optimal_cutoff = max(cutoff_profits, key=cutoff_profits.get)
max_profit = cutoff_profits[optimal_cutoff]
# Plotting
plt.figure(figsize=(10, 6))
plt.plot(list(cutoff_profits.keys()), list(cutoff_profits.values()), label='Total Profit')
plt.scatter(optimal_cutoff, max_profit, color='red', label='Optimal Cutoff')
plt.title('Optimal Intra-Day Profit Cutoff Analysis')
plt.xlabel('Profit Cutoff')
plt.ylabel('Total Profit')
plt.legend()
plt.grid(True)
plt.savefig('optimal_cutoff.png')
return optimal_cutoff, max_profit
# Example usage
# trades = [list of Trade objects]
if __name__ == '__main__':
# id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"]
# generate_trading_report_image(runner_ids=id_list)
batch_id = "90973e57"
optimal_cutoff, max_profit = find_optimal_cutoff(batch_id=batch_id)
print(f"Optimal Cutoff: {optimal_cutoff}, Max Profit: {max_profit}")

View File

@ -0,0 +1,167 @@
import matplotlib
import matplotlib.dates as mdates
#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg'
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from datetime import datetime
from typing import List
from enum import Enum
import numpy as np
import v2realbot.controller.services as cs
from rich import print
from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType
from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print
from pathlib import Path
from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide
from io import BytesIO
from v2realbot.utils.historicals import get_historical_bars
from alpaca.data.timeframe import TimeFrame, TimeFrameUnit
from collections import defaultdict
# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere
#LOSS and PROFIT without GRAPH
def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False):
#TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se
#TODO list of runner_ids
#TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera
if runner_ids is None and batch_id is None:
return -2, f"runner_id or batch_id must be present"
if batch_id is not None:
res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id)
if res != 0:
print(f"no batch {batch_id} found")
return -1, f"no batch {batch_id} found"
trades = []
cnt_max = len(runner_ids)
cnt = 0
#zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch
end_date = None
start_date = None
for id in runner_ids:
cnt += 1
#get runner
res, sada =cs.get_archived_runner_header_byID(id)
if res != 0:
print(f"no runner {id} found")
return -1, f"no runner {id} found"
print("archrunner")
#print(sada)
if cnt == 1:
start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started
if cnt == cnt_max:
end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped
# Parse trades
trades_dicts = sada.metrics["prescr_trades"]
for trade_dict in trades_dicts:
trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None
trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None
trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None
trades.append(Trade(**trade_dict))
#print(trades)
# symbol = sada.symbol
# #hour bars for backtested period
# print(start_date,end_date)
# bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour)
# print("bars for given period",bars)
# """Bars a dictionary with the following keys:
# * high: A list of high prices
# * low: A list of low prices
# * volume: A list of volumes
# * close: A list of close prices
# * hlcc4: A list of HLCC4 indicators
# * open: A list of open prices
# * time: A list of times in UTC (ISO 8601 format)
# * trades: A list of number of trades
# * resolution: A list of resolutions (all set to 'D')
# * confirmed: A list of booleans (all set to True)
# * vwap: A list of VWAP indicator
# * updated: A list of booleans (all set to True)
# * index: A list of integers (from 0 to the length of the list of daily bars)
# """
# Filter to only use trades with status 'CLOSED'
closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED]
#print(closed_trades)
if len(closed_trades) == 0:
return -1, "image generation no closed trades"
# Group trades by date and calculate daily profits
trades_by_day = defaultdict(list)
for trade in trades:
if trade.status == TradeStatus.CLOSED and trade.exit_time:
trade_day = trade.exit_time.date()
trades_by_day[trade_day].append(trade)
# Define ranges for loss and profit cutoffs
min_profit = 0
max_profit = 1000
profit_cutoffs = np.linspace(min_profit, max_profit, 20)
min_loss = 0
max_loss = -1000
loss_cutoffs = np.linspace(min_loss, max_loss, 20)
#print(profit_cutoffs, loss_cutoffs)
# Function to calculate total profit for a given cutoff
def calculate_total_profit(profit_cutoff, loss_cutoff):
total_profit = 0
for day, day_trades in trades_by_day.items():
daily_profit = 0
for trade in day_trades:
daily_profit += trade.profit
if daily_profit >= profit_cutoff or daily_profit <= loss_cutoff:
break
total_profit += daily_profit
return total_profit
# Initialize variables to store optimal cutoffs and max total profit
optimal_profit_cutoff = max_profit
optimal_loss_cutoff = min_loss
max_total_profit = float('-inf')
# Initialize a matrix to store total profits for each combination of cutoffs
total_profits_matrix = np.zeros((len(profit_cutoffs), len(loss_cutoffs)))
for i, profit_cutoff in enumerate(profit_cutoffs):
for j, loss_cutoff in enumerate(loss_cutoffs):
total_profit = calculate_total_profit(profit_cutoff, loss_cutoff)
total_profits_matrix[i, j] = total_profit
if total_profit > max_total_profit:
max_total_profit = total_profit
optimal_profit_cutoff = profit_cutoff
optimal_loss_cutoff = loss_cutoff
# Plotting the results as a heatmap
plt.figure(figsize=(20, 15))
sns.heatmap(total_profits_matrix, xticklabels=np.round(loss_cutoffs, 2), yticklabels=np.round(profit_cutoffs, 2),
annot=True, fmt=".0f", cmap="viridis") #fmt=".0f",
#plasma, coolmap, inferno, viridis
plt.title("Total Profit for Combinations of Profit and Loss Cutoffs")
plt.xlabel("Loss Cutoff")
plt.ylabel("Profit Cutoff")
plt.savefig('optimal_cutoff.png')
return optimal_profit_cutoff, optimal_loss_cutoff, max_total_profit
# Example usage
# trades = [list of Trade objects]
if __name__ == '__main__':
# id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"]
# generate_trading_report_image(runner_ids=id_list)
batch_id = "c76b4414"
optimal_profit_cutoff, optimal_loss_cutoff, max_profit = find_optimal_cutoff(batch_id=batch_id)
print(f"Optimal Profit Cutoff: {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_profit}")

View File

@ -0,0 +1,169 @@
import matplotlib
import matplotlib.dates as mdates
#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg'
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from datetime import datetime
from typing import List
from enum import Enum
import numpy as np
import v2realbot.controller.services as cs
from rich import print
from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType
from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print
from pathlib import Path
from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide
from io import BytesIO
from v2realbot.utils.historicals import get_historical_bars
from alpaca.data.timeframe import TimeFrame, TimeFrameUnit
from collections import defaultdict
# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere
#LOSS and PROFIT without GRAPH
def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False):
#TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se
#TODO list of runner_ids
#TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera
if runner_ids is None and batch_id is None:
return -2, f"runner_id or batch_id must be present"
if batch_id is not None:
res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id)
if res != 0:
print(f"no batch {batch_id} found")
return -1, f"no batch {batch_id} found"
trades = []
cnt_max = len(runner_ids)
cnt = 0
#zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch
end_date = None
start_date = None
for id in runner_ids:
cnt += 1
#get runner
res, sada =cs.get_archived_runner_header_byID(id)
if res != 0:
print(f"no runner {id} found")
return -1, f"no runner {id} found"
print("archrunner")
#print(sada)
if cnt == 1:
start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started
if cnt == cnt_max:
end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped
# Parse trades
trades_dicts = sada.metrics["prescr_trades"]
for trade_dict in trades_dicts:
trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None
trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None
trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None
trades.append(Trade(**trade_dict))
#print(trades)
# symbol = sada.symbol
# #hour bars for backtested period
# print(start_date,end_date)
# bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour)
# print("bars for given period",bars)
# """Bars a dictionary with the following keys:
# * high: A list of high prices
# * low: A list of low prices
# * volume: A list of volumes
# * close: A list of close prices
# * hlcc4: A list of HLCC4 indicators
# * open: A list of open prices
# * time: A list of times in UTC (ISO 8601 format)
# * trades: A list of number of trades
# * resolution: A list of resolutions (all set to 'D')
# * confirmed: A list of booleans (all set to True)
# * vwap: A list of VWAP indicator
# * updated: A list of booleans (all set to True)
# * index: A list of integers (from 0 to the length of the list of daily bars)
# """
# Filter to only use trades with status 'CLOSED'
closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED]
print(closed_trades)
if len(closed_trades) == 0:
return -1, "image generation no closed trades"
# Group trades by date and calculate daily profits
trades_by_day = defaultdict(list)
for trade in trades:
if trade.status == TradeStatus.CLOSED and trade.exit_time:
trade_day = trade.exit_time.date()
trades_by_day[trade_day].append(trade)
# Define ranges for loss and profit cutoffs
min_profit = 50
max_profit = 700 # Set an upper bound based on your data
profit_cutoffs = np.linspace(min_profit, max_profit, 50) # Adjust number of points as needed
min_loss = -50
max_loss = -700 # Assuming losses are negative values
loss_cutoffs = np.linspace(min_loss, max_loss, 50)
def calculate_total_profit(profit_cutoff, loss_cutoff):
total_profit = 0
for day, day_trades in trades_by_day.items():
daily_profit = 0
for trade in day_trades:
daily_profit += trade.profit
if daily_profit >= profit_cutoff or daily_profit <= loss_cutoff:
break
total_profit += daily_profit
return total_profit
# Evaluate each combination of cutoffs
optimal_profit_cutoff = max_profit
optimal_loss_cutoff = min_loss
max_total_profit = float('-inf')
for profit_cutoff in profit_cutoffs:
for loss_cutoff in loss_cutoffs:
total_profit = calculate_total_profit(profit_cutoff, loss_cutoff)
if total_profit > max_total_profit:
max_total_profit = total_profit
optimal_profit_cutoff = profit_cutoff
optimal_loss_cutoff = loss_cutoff
print(f"Optimal Profit Cutoff: {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_total_profit}")
# Optional: Plot the results or return them for further analysis
return optimal_profit_cutoff, optimal_loss_cutoff, max_total_profit
# # Plotting
# plt.figure(figsize=(10, 6))
# plt.plot(list(cutoff_profits.keys()), list(cutoff_profits.values()), label='Total Profit')
# plt.scatter(optimal_cutoff, max_profit, color='red', label='Optimal Cutoff')
# plt.title('Optimal Intra-Day Profit Cutoff Analysis')
# plt.xlabel('Profit Cutoff')
# plt.ylabel('Total Profit')
# plt.legend()
# plt.grid(True)
# plt.savefig('optimal_cutoff.png')
# return optimal_cutoff, max_profit
# Example usage
# trades = [list of Trade objects]
if __name__ == '__main__':
# id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"]
# generate_trading_report_image(runner_ids=id_list)
batch_id = "c76b4414"
optimal_profit_cutoff, optimal_loss_cutoff, max_profit = find_optimal_cutoff(batch_id=batch_id)
print(f"Optimal Profit Cutoff: {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_profit}")

View File

@ -0,0 +1,241 @@
import matplotlib
import matplotlib.dates as mdates
#matplotlib.use('Agg') # Set the Matplotlib backend to 'Agg'
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from datetime import datetime
from typing import List
from enum import Enum
import numpy as np
import v2realbot.controller.services as cs
from rich import print
from v2realbot.common.PrescribedTradeModel import TradeDirection, TradeStatus, Trade, TradeStoplossType
from v2realbot.utils.utils import isrising, isfalling,zoneNY, price2dec, safe_get#, print
from pathlib import Path
from v2realbot.config import WEB_API_KEY, DATA_DIR, MEDIA_DIRECTORY
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account, OrderSide
from io import BytesIO
from v2realbot.utils.historicals import get_historical_bars
from alpaca.data.timeframe import TimeFrame, TimeFrameUnit
from collections import defaultdict
from scipy.stats import zscore
from io import BytesIO
# Assuming Trade, TradeStatus, TradeDirection, TradeStoplossType classes are defined elsewhere
#LOSS and PROFIT without GRAPH
def find_optimal_cutoff(runner_ids: list = None, batch_id: str = None, stream: bool = False, rem_outliers:bool = False, file: str = "optimalcutoff.png",steps:int = 50):
#TODO dopracovat drawdown a minimalni a maximalni profity nikoliv cumulovane, zamyslet se
#TODO list of runner_ids
#TODO pridelat na vytvoreni runnera a batche, samostatne REST API + na remove archrunnera
if runner_ids is None and batch_id is None:
return -2, f"runner_id or batch_id must be present"
if batch_id is not None:
res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id)
if res != 0:
print(f"no batch {batch_id} found")
return -1, f"no batch {batch_id} found"
trades = []
cnt_max = len(runner_ids)
cnt = 0
#zatim zjistujeme start a end z min a max dni - jelikoz muze byt i seznam runner_ids a nejenom batch
end_date = None
start_date = None
for id in runner_ids:
cnt += 1
#get runner
res, sada =cs.get_archived_runner_header_byID(id)
if res != 0:
print(f"no runner {id} found")
return -1, f"no runner {id} found"
#print("archrunner")
#print(sada)
if cnt == 1:
start_date = sada.bt_from if sada.mode in [Mode.BT,Mode.PREP] else sada.started
if cnt == cnt_max:
end_date = sada.bt_to if sada.mode in [Mode.BT or Mode.PREP] else sada.stopped
# Parse trades
trades_dicts = sada.metrics["prescr_trades"]
for trade_dict in trades_dicts:
trade_dict['last_update'] = datetime.fromtimestamp(trade_dict.get('last_update')).astimezone(zoneNY) if trade_dict['last_update'] is not None else None
trade_dict['entry_time'] = datetime.fromtimestamp(trade_dict.get('entry_time')).astimezone(zoneNY) if trade_dict['entry_time'] is not None else None
trade_dict['exit_time'] = datetime.fromtimestamp(trade_dict.get('exit_time')).astimezone(zoneNY) if trade_dict['exit_time'] is not None else None
trades.append(Trade(**trade_dict))
#print(trades)
# symbol = sada.symbol
# #hour bars for backtested period
# print(start_date,end_date)
# bars= get_historical_bars(symbol, start_date, end_date, TimeFrame.Hour)
# print("bars for given period",bars)
# """Bars a dictionary with the following keys:
# * high: A list of high prices
# * low: A list of low prices
# * volume: A list of volumes
# * close: A list of close prices
# * hlcc4: A list of HLCC4 indicators
# * open: A list of open prices
# * time: A list of times in UTC (ISO 8601 format)
# * trades: A list of number of trades
# * resolution: A list of resolutions (all set to 'D')
# * confirmed: A list of booleans (all set to True)
# * vwap: A list of VWAP indicator
# * updated: A list of booleans (all set to True)
# * index: A list of integers (from 0 to the length of the list of daily bars)
# """
# Filter to only use trades with status 'CLOSED'
closed_trades = [trade for trade in trades if trade.status == TradeStatus.CLOSED]
#print(closed_trades)
if len(closed_trades) == 0:
return -1, "image generation no closed trades"
# # Group trades by date and calculate daily profits
# trades_by_day = defaultdict(list)
# for trade in trades:
# if trade.status == TradeStatus.CLOSED and trade.exit_time:
# trade_day = trade.exit_time.date()
# trades_by_day[trade_day].append(trade)
# Precompute daily cumulative profits
daily_cumulative_profits = defaultdict(list)
for trade in trades:
if trade.status == TradeStatus.CLOSED and trade.exit_time:
day = trade.exit_time.date()
daily_cumulative_profits[day].append(trade.profit)
for day in daily_cumulative_profits:
daily_cumulative_profits[day] = np.cumsum(daily_cumulative_profits[day])
if rem_outliers:
# Remove outliers based on z-scores
def remove_outliers(cumulative_profits):
all_profits = [profit[-1] for profit in cumulative_profits.values() if len(profit) > 0]
z_scores = zscore(all_profits)
print(z_scores)
filtered_profits = {}
for day, profits in cumulative_profits.items():
if len(profits) > 0:
day_z_score = z_scores[list(cumulative_profits.keys()).index(day)]
if abs(day_z_score) < 3: # Adjust threshold as needed
filtered_profits[day] = profits
return filtered_profits
daily_cumulative_profits = remove_outliers(daily_cumulative_profits)
# OPT1 Dynamically calculate profit_range and loss_range - based on eod daily profit
# all_final_profits = [profits[-1] for profits in daily_cumulative_profits.values() if len(profits) > 0]
# max_profit = max(all_final_profits)
# min_profit = min(all_final_profits)
# profit_range = (0, max_profit) if max_profit > 0 else (0, 0)
# loss_range = (min_profit, 0) if min_profit < 0 else (0, 0)
# OPT2 Calculate profit_range and loss_range based on all cumulative profits
all_cumulative_profits = np.concatenate([profits for profits in daily_cumulative_profits.values()])
max_cumulative_profit = np.max(all_cumulative_profits)
min_cumulative_profit = np.min(all_cumulative_profits)
profit_range = (0, max_cumulative_profit) if max_cumulative_profit > 0 else (0, 0)
loss_range = (min_cumulative_profit, 0) if min_cumulative_profit < 0 else (0, 0)
print("Calculated ranges", profit_range, loss_range)
num_points = steps # Adjust for speed vs accuracy
profit_cutoffs = np.linspace(*profit_range, num_points)
loss_cutoffs = np.linspace(*loss_range, num_points)
# OPT 3Statically define ranges for loss and profit cutoffs
# profit_range = (0, 1000) # Adjust based on your data
# loss_range = (-1000, 0)
# num_points = 20 # Adjust for speed vs accuracy
profit_cutoffs = np.linspace(*profit_range, num_points)
loss_cutoffs = np.linspace(*loss_range, num_points)
total_profits_matrix = np.zeros((len(profit_cutoffs), len(loss_cutoffs)))
for i, profit_cutoff in enumerate(profit_cutoffs):
for j, loss_cutoff in enumerate(loss_cutoffs):
total_profit = 0
for daily_profit in daily_cumulative_profits.values():
cutoff_index = np.where((daily_profit >= profit_cutoff) | (daily_profit <= loss_cutoff))[0]
if cutoff_index.size > 0:
total_profit += daily_profit[cutoff_index[0]]
else:
total_profit += daily_profit[-1] if daily_profit.size > 0 else 0
total_profits_matrix[i, j] = total_profit
# Find the optimal combination
optimal_idx = np.unravel_index(total_profits_matrix.argmax(), total_profits_matrix.shape)
optimal_profit_cutoff = profit_cutoffs[optimal_idx[0]]
optimal_loss_cutoff = loss_cutoffs[optimal_idx[1]]
max_profit = total_profits_matrix[optimal_idx]
# Plotting
# Setting up dark mode for the plots
plt.style.use('dark_background')
# Optionally, you can further customize colors, labels, and axes
params = {
'axes.titlesize': 9,
'axes.labelsize': 8,
'xtick.labelsize': 9,
'ytick.labelsize': 9,
'axes.labelcolor': '#a9a9a9', #a1a3aa',
'axes.facecolor': '#121722', #'#0e0e0e', #202020', # Dark background for plot area
'axes.grid': False, # Turn off the grid globally
'grid.color': 'gray', # If the grid is on, set grid line color
'grid.linestyle': '--', # Grid line style
'grid.linewidth': 1,
'xtick.color': '#a9a9a9',
'ytick.color': '#a9a9a9',
'axes.edgecolor': '#a9a9a9'
}
plt.rcParams.update(params)
plt.figure(figsize=(10, 8))
sns.heatmap(total_profits_matrix, xticklabels=np.rint(loss_cutoffs).astype(int), yticklabels=np.rint(profit_cutoffs).astype(int), cmap="viridis")
plt.xticks(rotation=90) # Rotate x-axis labels to be vertical
plt.yticks(rotation=0) # Keep y-axis labels horizontal
plt.gca().invert_yaxis()
plt.gca().invert_xaxis()
plt.suptitle("Total Profit for Combinations of Profit and Loss Cutoffs", fontsize=16)
plt.title(f"Optimal Profit Cutoff: {optimal_profit_cutoff:.2f}, Optimal Loss Cutoff: {optimal_loss_cutoff:.2f}, Max Profit: {max_profit:.2f}", fontsize=10)
plt.xlabel("Loss Cutoff")
plt.ylabel("Profit Cutoff")
if stream is False:
plt.savefig(file)
plt.close()
print(f"Optimal Profit Cutoff(rem_outliers:{rem_outliers}): {optimal_profit_cutoff}, Optimal Loss Cutoff: {optimal_loss_cutoff}, Max Profit: {max_profit}")
return 0, None
else:
# Return the image as a BytesIO stream
img_stream = BytesIO()
plt.savefig(img_stream, format='png')
plt.close()
img_stream.seek(0) # Rewind the stream to the beginning
return 0, img_stream
# Example usage
# trades = [list of Trade objects]
if __name__ == '__main__':
# id_list = ["e8938b2e-8462-441a-8a82-d823c6a025cb"]
# generate_trading_report_image(runner_ids=id_list)
batch_id = "c76b4414"
res, val = find_optimal_cutoff(batch_id=batch_id, file="optimal_cutoff_vectorized.png",steps=20)
#res, val = find_optimal_cutoff(batch_id=batch_id, rem_outliers=True, file="optimal_cutoff_vectorized_nooutliers.png")
print(res,val)

View File

@ -310,6 +310,7 @@
<button id="button_export_xml" class="btn btn-outline-success btn-sm">Export xml</button>
<button id="button_export_csv" class="btn btn-outline-success btn-sm">Export csv</button>
<button id="button_report" class="btn btn-outline-success btn-sm">Report(q)</button>
<button id="button_analyze" class="btn btn-outline-success btn-sm">Optimize Batch</button>
<!-- <button id="button_stopall" class="btn btn-outline-success btn-sm">Stop All</button>
<button id="button_refresh" class="btn btn-outline-success btn-sm">Refresh</button> -->
</div>

View File

@ -210,6 +210,7 @@ $(document).ready(function () {
$('#button_runagain_arch').attr('disabled','disabled');
$('#button_show_arch').attr('disabled','disabled');
$('#button_delete_arch').attr('disabled','disabled');
$('#button_analyze').attr('disabled','disabled');
$('#button_edit_arch').attr('disabled','disabled');
$('#button_compare_arch').attr('disabled','disabled');
@ -218,6 +219,7 @@ $(document).ready(function () {
if ($(this).hasClass('selected')) {
//$(this).removeClass('selected');
$('#button_show_arch').attr('disabled','disabled');
$('#button_analyze').attr('disabled','disabled');
$('#button_runagain_arch').attr('disabled','disabled');
$('#button_delete_arch').attr('disabled','disabled');
$('#button_edit_arch').attr('disabled','disabled');
@ -225,6 +227,7 @@ $(document).ready(function () {
} else {
//archiveRecords.$('tr.selected').removeClass('selected');
$(this).addClass('selected');
$('#button_analyze').attr('disabled',false);
$('#button_show_arch').attr('disabled',false);
$('#button_runagain_arch').attr('disabled',false);
$('#button_delete_arch').attr('disabled',false);
@ -413,12 +416,50 @@ $(document).ready(function () {
}
});
//generate batch optimization cutoff (predelat na button pro obecne analyzy batche)
$('#button_analyze').click(function () {
row = archiveRecords.row('.selected').data();
if (row == undefined || row.batch_id == undefined) {
return
}
console.log(row)
$('#button_analyze').attr('disabled','disabled');
$.ajax({
url:"/batches/optimizecutoff/"+row.batch_id,
beforeSend: function (xhr) {
xhr.setRequestHeader('X-API-Key',
API_KEY); },
method:"POST",
xhrFields: {
responseType: 'blob'
},
contentType: "application/json",
processData: false,
data: JSON.stringify(row.batch_id),
success:function(blob){
var url = window.URL || window.webkitURL;
console.log("vraceny obraz", blob)
console.log("url",url.createObjectURL(blob))
display_image(url.createObjectURL(blob))
$('#button_analyze').attr('disabled',false);
},
error: function(xhr, status, error) {
console.log("proc to skace do erroru?")
//window.alert(JSON.stringify(xhr));
console.log(JSON.stringify(xhr));
$('#button_analyze').attr('disabled',false);
}
})
});
//generate report button
$('#button_report').click(function () {
rows = archiveRecords.rows('.selected');
if (rows == undefined) {
return
}
$('#button_report').attr('disabled','disabled');
runnerIds = []
if(rows.data().length > 0 ) {
// Loop through the selected rows and display an alert with each row's ID
@ -444,11 +485,13 @@ $(document).ready(function () {
console.log("vraceny obraz", blob)
console.log("url",url.createObjectURL(blob))
display_image(url.createObjectURL(blob))
$('#button_report').attr('disabled',false);
},
error: function(xhr, status, error) {
console.log("proc to skace do erroru?")
//window.alert(JSON.stringify(xhr));
console.log(JSON.stringify(xhr));
$('#button_report').attr('disabled',false);
}
})
});