From 8f420258b56d433d392e45a616e7a5d723611007 Mon Sep 17 00:00:00 2001 From: David Brazda Date: Wed, 13 Mar 2024 12:26:49 +0100 Subject: [PATCH] #168 #166 and additional fixes --- testy/archive/alpacaGetHistoryBars.py | 12 +-- v2realbot/ENTRY_ClassicSL_v01.py | 3 + v2realbot/backtesting/backtester.py | 24 +++--- v2realbot/common/model.py | 86 +++++++++++-------- v2realbot/controller/services.py | 2 +- v2realbot/interfaces/live_interface.py | 29 +++++-- v2realbot/main.py | 3 +- v2realbot/strategy/StrategyClassicSL.py | 18 ++-- .../newtrade/prescribedtrades.py | 1 + v2realbot/strategyblocks/newtrade/sizing.py | 13 ++- v2realbot/utils/utils.py | 10 +++ 11 files changed, 130 insertions(+), 71 deletions(-) diff --git a/testy/archive/alpacaGetHistoryBars.py b/testy/archive/alpacaGetHistoryBars.py index 559e63b..8ed60f3 100644 --- a/testy/archive/alpacaGetHistoryBars.py +++ b/testy/archive/alpacaGetHistoryBars.py @@ -23,12 +23,12 @@ clientTrading = TradingClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, #get previous days bar -datetime_object_from = datetime.datetime(2023, 10, 11, 4, 0, 00, tzinfo=datetime.timezone.utc) -datetime_object_to = datetime.datetime(2023, 10, 16, 16, 1, 00, tzinfo=datetime.timezone.utc) -calendar_request = GetCalendarRequest(start=datetime_object_from,end=datetime_object_to) -cal_dates = clientTrading.get_calendar(calendar_request) -print(cal_dates) -bar_request = StockBarsRequest(symbol_or_symbols="BAC",timeframe=TimeFrame.Day, start=datetime_object_from, end=datetime_object_to, feed=DataFeed.SIP) +datetime_object_from = datetime.datetime(2024, 3, 9, 13, 29, 00, tzinfo=datetime.timezone.utc) +datetime_object_to = datetime.datetime(2024, 3, 11, 20, 1, 00, tzinfo=datetime.timezone.utc) +# calendar_request = GetCalendarRequest(start=datetime_object_from,end=datetime_object_to) +# cal_dates = clientTrading.get_calendar(calendar_request) +# print(cal_dates) +bar_request = StockBarsRequest(symbol_or_symbols="BAC",timeframe=TimeFrame.Minute, start=datetime_object_from, end=datetime_object_to, feed=DataFeed.SIP) # bars = client.get_stock_bars(bar_request).df diff --git a/v2realbot/ENTRY_ClassicSL_v01.py b/v2realbot/ENTRY_ClassicSL_v01.py index 91856df..349e549 100644 --- a/v2realbot/ENTRY_ClassicSL_v01.py +++ b/v2realbot/ENTRY_ClassicSL_v01.py @@ -115,6 +115,9 @@ def init(state: StrategyState): #models state.vars.loaded_models = {} + #state attributes for martingale sizing mngmt + state.vars["martingale"] = dict(cont_loss_series_cnt=0) + #INITIALIZE CBAR INDICATORS - do vlastni funkce #state.cbar_indicators['ivwap'] = [] state.vars.last_tick_price = 0 diff --git a/v2realbot/backtesting/backtester.py b/v2realbot/backtesting/backtester.py index 1cb7e4e..e05b901 100644 --- a/v2realbot/backtesting/backtester.py +++ b/v2realbot/backtesting/backtester.py @@ -40,7 +40,7 @@ from uuid import UUID, uuid4 from alpaca.trading.enums import OrderSide, OrderStatus, TradeEvent, OrderType from v2realbot.common.model import TradeUpdate, Order -#from rich import print +from rich import print as printanyway import threading import asyncio from v2realbot.config import DATA_DIR @@ -479,11 +479,11 @@ class Backtester: print("BT: submit order entry") if not time or time < 0: - print("time musi byt vyplneny") + printanyway("time musi byt vyplneny") return -1 if not size or int(size) < 0: - print("size musi byt vetsi nez 0") + printanyway("size musi byt vetsi nez 0") return -1 if (order_type != OrderType.MARKET) and (order_type != OrderType.LIMIT): @@ -491,11 +491,11 @@ class Backtester: return -1 if not side == OrderSide.BUY and not side == OrderSide.SELL: - print("side buy/sell required") + printanyway("side buy/sell required") return -1 if order_type == OrderType.LIMIT and count_decimals(price) > 2: - print("only 2 decimals supported", price) + printanyway("only 2 decimals supported", price) return -1 #pokud neexistuje klic v accountu vytvorime si ho @@ -517,14 +517,14 @@ class Backtester: actual_minus_reserved = int(self.account[symbol][0]) - reserved if actual_minus_reserved > 0 and actual_minus_reserved - int(size) < 0: - print("not enough shares available to sell or shorting while long position",self.account[symbol][0],"reserved",reserved,"available",int(self.account[symbol][0]) - reserved,"selling",size) + printanyway("not enough shares available to sell or shorting while long position",self.account[symbol][0],"reserved",reserved,"available",int(self.account[symbol][0]) - reserved,"selling",size) return -1 #if is shorting - check available cash to short if actual_minus_reserved <= 0: cena = price if price else self.get_last_price(time, self.symbol) if (self.cash - reserved_price - float(int(size)*float(cena))) < 0: - print("not enough cash for shorting. cash",self.cash,"reserved",reserved,"available",self.cash-reserved,"needed",float(int(size)*float(cena))) + printanyway("not enough cash for shorting. cash",self.cash,"reserved",reserved,"available",self.cash-reserved,"needed",float(int(size)*float(cena))) return -1 #check for available cash @@ -543,14 +543,14 @@ class Backtester: #jde o uzavreni shortu if actual_plus_reserved_qty < 0 and (actual_plus_reserved_qty + int(size)) > 0: - print("nejprve je treba uzavrit short pozici pro buy res_qty, size", actual_plus_reserved_qty, size) + printanyway("nejprve je treba uzavrit short pozici pro buy res_qty, size", actual_plus_reserved_qty, size) return -1 #jde o standardni long, kontroluju cash if actual_plus_reserved_qty >= 0: cena = price if price else self.get_last_price(time, self.symbol) if (self.cash - reserved_price - float(int(size)*float(cena))) < 0: - print("not enough cash to buy long. cash",self.cash,"reserved_qty",reserved_qty,"reserved_price",reserved_price, "available",self.cash-reserved_price,"needed",float(int(size)*float(cena))) + printanyway("not enough cash to buy long. cash",self.cash,"reserved_qty",reserved_qty,"reserved_price",reserved_price, "available",self.cash-reserved_price,"needed",float(int(size)*float(cena))) return -1 id = str(uuid4()) @@ -577,11 +577,11 @@ class Backtester: print("BT: replace order entry",id,size,price) if not price and not size: - print("size or price required") + printanyway("size or price required") return -1 if len(self.open_orders) == 0: - print("BT: order doesnt exist") + printanyway("BT: order doesnt exist") return 0 #with lock: for o in self.open_orders: @@ -609,7 +609,7 @@ class Backtester: """ print("BT: cancel order entry",id) if len(self.open_orders) == 0: - print("BTC: order doesnt exist") + printanyway("BTC: order doesnt exist") return 0 #with lock: for o in self.open_orders: diff --git a/v2realbot/common/model.py b/v2realbot/common/model.py index e26a475..78f4f17 100644 --- a/v2realbot/common/model.py +++ b/v2realbot/common/model.py @@ -94,12 +94,12 @@ class TestList(BaseModel): class Trade(BaseModel): symbol: str timestamp: datetime - exchange: Optional[Union[Exchange, str]] + exchange: Optional[Union[Exchange, str]] = None price: float size: float id: int - conditions: Optional[List[str]] - tape: Optional[str] + conditions: Optional[List[str]] = None + tape: Optional[str] = None #persisted object in pickle @@ -114,8 +114,20 @@ class StrategyInstance(BaseModel): close_rush: int = 0 stratvars_conf: str add_data_conf: str - note: Optional[str] - history: Optional[str] + note: Optional[str] = None + history: Optional[str] = None + + def __setstate__(self, state: dict[Any, Any]) -> None: + """ + Hack to allow unpickling models stored from pydantic V1 + """ + state.setdefault("__pydantic_extra__", {}) + state.setdefault("__pydantic_private__", {}) + + if "__pydantic_fields_set__" not in state: + state["__pydantic_fields_set__"] = state.get("__fields_set__") + + super().__setstate__(state) class RunRequest(BaseModel): id: UUID @@ -125,8 +137,8 @@ class RunRequest(BaseModel): debug: bool = False strat_json: Optional[str] = None ilog_save: bool = False - bt_from: datetime = None - bt_to: datetime = None + bt_from: Optional[datetime] = None + bt_to: Optional[datetime] = None #weekdays filter #pokud je uvedeny filtrujeme tyto dny weekdays_filter: Optional[list] = None @@ -147,8 +159,8 @@ class RunManagerRecord(BaseModel): mode: Mode note: Optional[str] = None ilog_save: bool = False - bt_from: datetime = None - bt_to: datetime = None + bt_from: Optional[datetime] = None + bt_to: Optional[datetime] = None #weekdays filter #pokud je uvedeny filtrujeme tyto dny weekdays_filter: Optional[list] = None #list of strings 0-6 representing days to run @@ -156,9 +168,9 @@ class RunManagerRecord(BaseModel): batch_id: Optional[str] = None testlist_id: Optional[str] = None start_time: str #time (HH:MM) that start function is called - stop_time: Optional[str] #time (HH:MM) that stop function is called + stop_time: Optional[str] = None #time (HH:MM) that stop function is called status: SchedulerStatus - last_processed: Optional[datetime] + last_processed: Optional[datetime] = None history: Optional[str] = None valid_from: Optional[datetime] = None # US East time zone daetime valid_to: Optional[datetime] = None # US East time zone daetime @@ -193,10 +205,10 @@ class Runner(BaseModel): run_name: Optional[str] = None run_note: Optional[str] = None run_ilog_save: Optional[bool] = False - run_trade_count: Optional[int] - run_profit: Optional[float] - run_positions: Optional[int] - run_avgp: Optional[float] + run_trade_count: Optional[int] = None + run_profit: Optional[float] = None + run_positions: Optional[int] = None + run_avgp: Optional[float] = None run_strat_json: Optional[str] = None run_stopped: Optional[datetime] = None run_paused: Optional[datetime] = None @@ -230,41 +242,41 @@ class Bar(BaseModel): low: float close: float volume: float - trade_count: Optional[float] - vwap: Optional[float] + trade_count: Optional[float] = 0 + vwap: Optional[float] = 0 class Order(BaseModel): id: UUID submitted_at: datetime - filled_at: Optional[datetime] - canceled_at: Optional[datetime] + filled_at: Optional[datetime] = None + canceled_at: Optional[datetime] = None symbol: str qty: int status: OrderStatus order_type: OrderType - filled_qty: Optional[int] - filled_avg_price: Optional[float] + filled_qty: Optional[int] = None + filled_avg_price: Optional[float] = None side: OrderSide - limit_price: Optional[float] + limit_price: Optional[float] = None #entita pro kazdy kompletni FILL, je navazana na prescribed_trade class TradeUpdate(BaseModel): event: Union[TradeEvent, str] - execution_id: Optional[UUID] + execution_id: Optional[UUID] = None order: Order timestamp: datetime - position_qty: Optional[float] - price: Optional[float] - qty: Optional[float] - value: Optional[float] - cash: Optional[float] - pos_avg_price: Optional[float] - profit: Optional[float] - profit_sum: Optional[float] - rel_profit: Optional[float] - rel_profit_cum: Optional[float] - signal_name: Optional[str] - prescribed_trade_id: Optional[str] + position_qty: Optional[float] = None + price: Optional[float] = None + qty: Optional[float] = None + value: Optional[float] = None + cash: Optional[float] = None + pos_avg_price: Optional[float] = None + profit: Optional[float] = None + profit_sum: Optional[float] = None + rel_profit: Optional[float] = None + rel_profit_cum: Optional[float] = None + signal_name: Optional[str] = None + prescribed_trade_id: Optional[str] = None class RunArchiveChange(BaseModel): @@ -332,7 +344,7 @@ class RunArchiveViewPagination(BaseModel): #trida pro ukladani historie stoplossy do ext_data class SLHistory(BaseModel): - id: Optional[UUID] + id: Optional[UUID] = None time: datetime sl_val: float @@ -345,7 +357,7 @@ class RunArchiveDetail(BaseModel): indicators: List[dict] statinds: dict trades: List[TradeUpdate] - ext_data: Optional[dict] + ext_data: Optional[dict] = None class InstantIndicator(BaseModel): diff --git a/v2realbot/controller/services.py b/v2realbot/controller/services.py index 2d1a8f0..75af547 100644 --- a/v2realbot/controller/services.py +++ b/v2realbot/controller/services.py @@ -1896,7 +1896,7 @@ def get_alpaca_history_bars(symbol: str, datetime_object_from: datetime, datetim # Workaround of error when no data foun d AttributeError and has the specific message if isinstance(e, AttributeError) and str(e) == "'NoneType' object has no attribute 'items'": print("Caught the specific AttributeError: 'NoneType' object has no attribute 'items' means NO DATA FOUND") - #print(str(e) + format_exc()) + print(str(e) + format_exc()) return 0, result else: print(str(e) + format_exc()) diff --git a/v2realbot/interfaces/live_interface.py b/v2realbot/interfaces/live_interface.py index 1da2572..8e23adc 100644 --- a/v2realbot/interfaces/live_interface.py +++ b/v2realbot/interfaces/live_interface.py @@ -40,7 +40,9 @@ class LiveInterface(GeneralInterface): return market_order.id except Exception as e: - print("Nepodarilo se odeslat buy", str(e)) + reason = "Nepodarilo se market buy:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) return -1 """buy limit""" @@ -65,7 +67,9 @@ class LiveInterface(GeneralInterface): return limit_order.id except Exception as e: - print("Nepodarilo se odeslat limitku", str(e)) + reason = "Nepodarilo se odeslat buy limitku:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) return -1 """sell market""" @@ -87,7 +91,9 @@ class LiveInterface(GeneralInterface): return market_order.id except Exception as e: - print("Nepodarilo se odeslat sell", str(e)) + reason = "Nepodarilo se odeslat sell:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) return -1 """sell limit""" @@ -112,8 +118,9 @@ class LiveInterface(GeneralInterface): return limit_order.id except Exception as e: - print("Nepodarilo se odeslat sell_l", str(e)) - #raise Exception(e) + reason = "Nepodarilo se odeslat sell limitku:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) return -1 """order replace""" @@ -136,7 +143,9 @@ class LiveInterface(GeneralInterface): if e.code == 42210000: return orderid else: ##mozna tady proste vracet vzdy ok - print("Neslo nahradit profitku. Problem",str(e)) + reason = "Neslo nahradit profitku. Problem:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) return -1 #raise Exception(e) @@ -150,7 +159,9 @@ class LiveInterface(GeneralInterface): #order doesnt exist if e.code == 40410000: return 0 else: - print("nepovedlo se zrusit objednavku", str(e)) + reason = "Nepovedlo se zrusit objednavku:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) #raise Exception(e) return -1 @@ -178,7 +189,9 @@ class LiveInterface(GeneralInterface): #list of Orders (orderlist[0].id) return orderlist except Exception as e: - print("Chyba pri dotazeni objednávek.", str(e)) + reason = "Chyba pri dotazeni objednávek:" + str(e) + format_exc() + print(reason) + send_to_telegram(reason) #raise Exception (e) return -1 diff --git a/v2realbot/main.py b/v2realbot/main.py index 2675e91..c09b96e 100644 --- a/v2realbot/main.py +++ b/v2realbot/main.py @@ -690,7 +690,8 @@ def _generate_analysis(analyzerInputs: AnalyzerInputs): if res == 0: return StreamingResponse(stream, media_type="image/png") elif res < 0: - raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {res}:{id}") + print("Error when generating analysis: ",str(stream)) + raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {res}:{stream}") except Exception as e: raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {str(e)}" + format_exc()) diff --git a/v2realbot/strategy/StrategyClassicSL.py b/v2realbot/strategy/StrategyClassicSL.py index 0431c9f..369d694 100644 --- a/v2realbot/strategy/StrategyClassicSL.py +++ b/v2realbot/strategy/StrategyClassicSL.py @@ -9,7 +9,7 @@ from alpaca.trading.enums import TradeEvent, OrderStatus from v2realbot.indicators.indicators import ema import orjson from datetime import datetime -#from rich import print +from rich import print as printanyway from random import randrange from alpaca.common.exceptions import APIError import numpy as np @@ -153,6 +153,10 @@ class StrategyClassicSL(Strategy): self.state.rel_profit_cum.append(rel_profit) rel_profit_cum_calculated = round(np.sum(self.state.rel_profit_cum),5) + #pro martingale updatujeme loss_series_cnt + self.state.vars["martingale"]["cont_loss_series_cnt"] = 0 if rel_profit > 0 else self.state.vars["martingale"]["cont_loss_series_cnt"]+1 + self.state.ilog(lvl=1, e=f"update cont_loss_series_cnt na {self.state.vars['martingale']['cont_loss_series_cnt']}") + self.state.ilog(e=f"BUY notif - SHORT PROFIT: {partial_exit=} {partial_last=} {round(float(trade_profit),3)} celkem:{round(float(self.state.profit),3)} rel:{float(rel_profit)} rel_cum:{round(rel_profit_cum_calculated,7)}", msg=str(data.event), rel_profit_cum=str(self.state.rel_profit_cum), bought_amount=bought_amount, avg_costs=avg_costs, trade_qty=data.qty, trade_price=data.price, orderid=str(data.order.id)) #zapsat profit do prescr.trades @@ -298,6 +302,10 @@ class StrategyClassicSL(Strategy): self.state.rel_profit_cum.append(rel_profit) rel_profit_cum_calculated = round(np.sum(self.state.rel_profit_cum),5) + #pro martingale updatujeme loss_series_cnt + self.state.vars["martingale"]["cont_loss_series_cnt"] = 0 if rel_profit > 0 else self.state.vars["martingale"]["cont_loss_series_cnt"]+1 + self.state.ilog(lvl=1, e=f"update cont_loss_series_cnt na {self.state.vars['martingale']['cont_loss_series_cnt']}") + self.state.ilog(e=f"SELL notif - LONG PROFIT {partial_exit=} {partial_last=}:{round(float(trade_profit),3)} celkem:{round(float(self.state.profit),3)} rel:{float(rel_profit)} rel_cum:{round(rel_profit_cum_calculated,7)}", msg=str(data.event), rel_profit_cum = str(self.state.rel_profit_cum), sold_amount=sold_amount, avg_costs=avg_costs, trade_qty=data.qty, trade_price=data.price, orderid=str(data.order.id)) #zapsat profit do prescr.trades @@ -423,12 +431,12 @@ class StrategyClassicSL(Strategy): #jde o uzavreni short pozice if int(self.state.positions) < 0 and (int(self.state.positions) + int(sizer)) > 0: self.state.ilog(e="buy nelze nakoupit vic nez shortuji", positions=self.state.positions, size=size) - print("buy nelze nakoupit vic nez shortuji") + printanyway("buy nelze nakoupit vic nez shortuji") return -2 if int(self.state.positions) >= self.state.vars.maxpozic: self.state.ilog(e="buy Maxim mnozstvi naplneno", positions=self.state.positions) - print("max mnostvi naplneno") + printanyway("max mnostvi naplneno") return 0 self.state.blockbuy = 1 @@ -447,13 +455,13 @@ class StrategyClassicSL(Strategy): #jde o uzavreni long pozice if int(self.state.positions) > 0 and (int(self.state.positions) - int(size)) < 0: self.state.ilog(e="nelze prodat vic nez longuji", positions=self.state.positions, size=size) - print("nelze prodat vic nez longuji") + printanyway("nelze prodat vic nez longuji") return -2 #pokud shortuji a mam max pozic if int(self.state.positions) < 0 and abs(int(self.state.positions)) >= self.state.vars.maxpozic: self.state.ilog(e="short - Maxim mnozstvi naplneno", positions=self.state.positions, size=size) - print("max mnostvi naplneno") + printanyway("short - Maxim mnozstvi naplneno") return 0 #self.state.blocksell = 1 diff --git a/v2realbot/strategyblocks/newtrade/prescribedtrades.py b/v2realbot/strategyblocks/newtrade/prescribedtrades.py index 54e4a36..99752e2 100644 --- a/v2realbot/strategyblocks/newtrade/prescribedtrades.py +++ b/v2realbot/strategyblocks/newtrade/prescribedtrades.py @@ -78,6 +78,7 @@ def execute_prescribed_trades(state: StrategyState, data): size = state.vars.chunk res = state.sell(size=size) if isinstance(res, int) and res < 0: + print(f"error in required operation SHORT {res}") raise Exception(f"error in required operation SHORT {res}") #defaultní goalprice nastavujeme az v notifikaci diff --git a/v2realbot/strategyblocks/newtrade/sizing.py b/v2realbot/strategyblocks/newtrade/sizing.py index f670842..7440172 100644 --- a/v2realbot/strategyblocks/newtrade/sizing.py +++ b/v2realbot/strategyblocks/newtrade/sizing.py @@ -147,7 +147,18 @@ def get_multiplier(state: StrategyState, data, signaloptions: dict, direction: T multiplier = f(input_value) state.ilog(lvl=1,e=f"SIZER - Interpolated value {multiplier}", input_value=input_value, pattern_source_axis=pattern_source_axis, pattern_size_axis=pattern_size_axis, options=options, time=state.time) - if multiplier > 1 or multiplier <= 0: + martingale_enabled = utls.safe_get(options, "martingale_enabled", False) + + #pocet ztrátových obchodů v řadě mi udává multiplikátor (0 - 1, 1 ztráta 2x, 3 v řadě - 4x atp.) + if martingale_enabled: + cont_loss_series_cnt = state.vars["martingale"]["cont_loss_series_cnt"] + if cont_loss_series_cnt == 0: + multiplier = 1 + else: + multiplier = 2 ** cont_loss_series_cnt + state.ilog(lvl=1,e=f"SIZER - MARTINGALE {multiplier}", options=options, time=state.time, cont_loss_series_cnt=cont_loss_series_cnt) + + if (martingale_enabled is False and multiplier > 1) or multiplier <= 0: state.ilog(lvl=1,e=f"SIZER - Mame nekde problem MULTIPLIER mimo RANGE ERROR {multiplier}", options=options, time=state.time) multiplier = 1 return multiplier diff --git a/v2realbot/utils/utils.py b/v2realbot/utils/utils.py index f076316..49018af 100644 --- a/v2realbot/utils/utils.py +++ b/v2realbot/utils/utils.py @@ -74,6 +74,16 @@ def fetch_calendar_data(start, end, max_retries=5, backoff_factor=1): :return: Calendar data. :raises: ConnectionError if all retries fail. """ + # Ensure start and end are of type datetime.date + if isinstance(start, datetime): + start = start.date() + if isinstance(end, datetime): + end = end.date() + + # Verify that start and end are datetime.date objects after conversion + if not all([isinstance(start, date), isinstance(end, date)]): + raise ValueError("start and end must be datetime.date objects") + clientTrading = TradingClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, raw_data=False) calendar_request = GetCalendarRequest(start=start, end=end) last_exception = None