toml validation to frontend

testy/archive/alpacaGetHistoryBars.py

@@ -23,12 +23,12 @@ clientTrading = TradingClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY,
 
 #get previous days bar
 
-datetime_object_from = datetime.datetime(2023, 10, 11, 4, 0, 00, tzinfo=datetime.timezone.utc)
-datetime_object_to = datetime.datetime(2023, 10, 16, 16, 1, 00, tzinfo=datetime.timezone.utc)
-calendar_request = GetCalendarRequest(start=datetime_object_from,end=datetime_object_to)
-cal_dates = clientTrading.get_calendar(calendar_request)
-print(cal_dates)
-bar_request = StockBarsRequest(symbol_or_symbols="BAC",timeframe=TimeFrame.Day, start=datetime_object_from, end=datetime_object_to, feed=DataFeed.SIP)
+datetime_object_from = datetime.datetime(2024, 3, 9, 13, 29, 00, tzinfo=datetime.timezone.utc)
+datetime_object_to = datetime.datetime(2024, 3, 11, 20, 1, 00, tzinfo=datetime.timezone.utc)
+# calendar_request = GetCalendarRequest(start=datetime_object_from,end=datetime_object_to)
+# cal_dates = clientTrading.get_calendar(calendar_request)
+# print(cal_dates)
+bar_request = StockBarsRequest(symbol_or_symbols="BAC",timeframe=TimeFrame.Minute, start=datetime_object_from, end=datetime_object_to, feed=DataFeed.SIP)
 
 # bars = client.get_stock_bars(bar_request).df
 

testy/getrunnerdetail.py

@@ -0,0 +1,89 @@
+
+from v2realbot.common.model import RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest
+import v2realbot.controller.services as cs
+from v2realbot.utils.utils import slice_dict_lists,zoneUTC,safe_get, AttributeDict
+id = "b11c66d9-a9b6-475a-9ac1-28b11e1b4edf"
+state = AttributeDict(vars={})
+
+##základ pro init_attached_data in strategy.init
+
+# def get_previous_runner(state):
+#     runner : Runner
+#     res, runner = cs.get_runner(state.runner_id)
+#     if res < 0:
+#         print(f"Not running {id}")
+#         return 0, None
+    
+#     return 0, runner.batch_id
+
+def attach_previous_data(state):
+    runner : Runner
+    #get batch_id of current runer
+    res, runner = cs.get_runner(state.runner_id)
+    if res < 0 or runner.batch_id is None:
+        print(f"Couldnt get previous runner {val}")
+        return None
+    
+    batch_id = runner.batch_id
+    #batch_id = "6a6b0bcf"
+
+    res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id, "desc")
+    if res < 0:
+        msg = f"error whne fetching runners of batch {batch_id} {runner_ids}"
+        print(msg)
+        return None
+    
+    if runner_ids is None or len(runner_ids) == 0:
+        print(f"no runners found for batch {batch_id} {runner_ids}")
+        return None
+    
+    last_runner = runner_ids[0]
+    print("Previous runner identified:", last_runner)
+
+    #get details from the runner
+    res, val = cs.get_archived_runner_details_byID(last_runner)
+    if res < 0:
+        print(f"no archived runner {last_runner}")
+
+    detail = RunArchiveDetail(**val)
+    #print("toto jsme si dotahnuli", detail.bars)
+
+    # from stratvars directives 
+    attach_previous_bars_indicators = safe_get(state.vars, "attach_previous_bars_indicators", 50)
+    attach_previous_cbar_indicators = safe_get(state.vars, "attach_previous_cbar_indicators", 50)
+    # [stratvars]
+    #     attach_previous_bars_indicators = 50
+    #     attach_previous_cbar_indicators = 50
+
+    #indicators datetime utc
+    indicators = slice_dict_lists(d=detail.indicators[0],last_item=attach_previous_bars_indicators, time_to_datetime=True)
+
+    #time -datetime utc, updated - timestamp float
+    bars = slice_dict_lists(d=detail.bars, last_item=attach_previous_bars_indicators, time_to_datetime=True)
+
+    #cbar_indicatzors #float
+    cbar_inds = slice_dict_lists(d=detail.indicators[1],last_item=attach_previous_cbar_indicators)
+
+    #USE these as INITs - TADY SI TO JESTE ZASTAVIT a POROVNAT
+    print(f"{state.indicators=} NEW:{indicators=}")
+    state.indicators = indicators
+    print(f"{state.bars=} NEW:{bars=}")
+    state.bars = bars
+    print(f"{state.cbar_indicators=} NEW:{cbar_inds=}")
+    state.cbar_indicators = cbar_inds
+
+    print("BARS and INDS INITIALIZED")
+    #bars
+
+
+    #tady budou pripadne dalsi inicializace, z ext_data
+    print("EXT_DATA", detail.ext_data)
+    #podle urciteho nastaveni napr.v konfiguraci se pouziji urcite promenne
+
+    #pridavame dailyBars z extData
+    # if hasattr(detail, "ext_data") and "dailyBars" in detail.ext_data:
+    #     state.dailyBars = detail.ext_data["dailyBars"]
+
+
+if __name__ == "__main__":
+    attach_previous_data(state)

v2realbot/ENTRY_ClassicSL_v01.py

@@ -16,6 +16,7 @@ from v2realbot.strategyblocks.newtrade.signals import signal_search
 from v2realbot.strategyblocks.activetrade.activetrade_hub import manage_active_trade
 from v2realbot.strategyblocks.inits.init_indicators import initialize_dynamic_indicators
 from v2realbot.strategyblocks.inits.init_directives import intialize_directive_conditions
+from v2realbot.strategyblocks.inits.init_attached_data import attach_previous_data
 from alpaca.trading.client import TradingClient
 from v2realbot.config import ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, DATA_DIR
 from alpaca.trading.models import Calendar
@@ -115,6 +116,10 @@ def init(state: StrategyState):
     #models
     state.vars.loaded_models = {}
 
+    #state attributes for martingale sizing mngmt
+    state.vars["transferables"] = {}
+    state.vars["transferables"]["martingale"] = dict(cont_loss_series_cnt=0)
+    
     #INITIALIZE CBAR INDICATORS - do vlastni funkce
     #state.cbar_indicators['ivwap'] = []
     state.vars.last_tick_price = 0
@@ -128,6 +133,9 @@ def init(state: StrategyState):
     initialize_dynamic_indicators(state)
     intialize_directive_conditions(state)
 
+    #attach part of yesterdays data, bars, indicators, cbar_indicators
+    attach_previous_data(state)
+
     #intitialize indicator mapping (for use in operation) -  mozna presunout do samostatne funkce prip dat do base kdyz se osvedci
     local_dict_cbar_inds = {key: state.cbar_indicators[key] for key in state.cbar_indicators.keys() if key != "time"}
     local_dict_inds = {key: state.indicators[key] for key in state.indicators.keys() if key != "time"}

v2realbot/backtesting/backtester.py

@@ -40,7 +40,7 @@
 from uuid import UUID, uuid4
 from alpaca.trading.enums import OrderSide, OrderStatus, TradeEvent, OrderType
 from v2realbot.common.model import TradeUpdate, Order
-#from rich import print
+from rich import print as printanyway
 import threading
 import asyncio
 from v2realbot.config import DATA_DIR
@@ -479,11 +479,11 @@ class Backtester:
         print("BT: submit order entry")
 
         if not time or time < 0:
-            print("time musi byt vyplneny")
+            printanyway("time musi byt vyplneny")
             return -1
 
         if not size or int(size) < 0:
-            print("size musi byt vetsi nez 0")
+            printanyway("size musi byt vetsi nez 0")
             return -1
 
         if (order_type != OrderType.MARKET) and (order_type != OrderType.LIMIT):
@@ -491,11 +491,11 @@ class Backtester:
             return -1
 
         if not side == OrderSide.BUY and not side == OrderSide.SELL:
-            print("side buy/sell required")
+            printanyway("side buy/sell required")
             return -1
         
         if order_type == OrderType.LIMIT and count_decimals(price) > 2:
-            print("only 2 decimals supported", price)
+            printanyway("only 2 decimals supported", price)
             return -1
     
         #pokud neexistuje klic v accountu vytvorime si ho
@@ -517,14 +517,14 @@ class Backtester:
             
             actual_minus_reserved = int(self.account[symbol][0]) - reserved 
             if actual_minus_reserved > 0 and actual_minus_reserved - int(size) < 0:
-                print("not enough shares available to sell or shorting while long position",self.account[symbol][0],"reserved",reserved,"available",int(self.account[symbol][0]) - reserved,"selling",size)
+                printanyway("not enough shares available to sell or shorting while long position",self.account[symbol][0],"reserved",reserved,"available",int(self.account[symbol][0]) - reserved,"selling",size)
                 return -1
             
             #if is shorting - check available cash to short
             if actual_minus_reserved <= 0:
                 cena = price if price else self.get_last_price(time, self.symbol)
                 if (self.cash - reserved_price - float(int(size)*float(cena))) < 0:
-                    print("not enough cash for shorting. cash",self.cash,"reserved",reserved,"available",self.cash-reserved,"needed",float(int(size)*float(cena)))
+                    printanyway("not enough cash for shorting. cash",self.cash,"reserved",reserved,"available",self.cash-reserved,"needed",float(int(size)*float(cena)))
                     return -1               
 
         #check for available cash
@@ -543,14 +543,14 @@ class Backtester:
             
             #jde o uzavreni shortu
             if actual_plus_reserved_qty < 0 and (actual_plus_reserved_qty + int(size)) > 0:
-                print("nejprve je treba uzavrit short pozici pro buy res_qty, size", actual_plus_reserved_qty, size)
+                printanyway("nejprve je treba uzavrit short pozici pro buy res_qty, size", actual_plus_reserved_qty, size)
                 return -1
 
             #jde o standardni long, kontroluju cash
             if actual_plus_reserved_qty >= 0:
                 cena = price if price else self.get_last_price(time, self.symbol)
                 if (self.cash - reserved_price - float(int(size)*float(cena))) < 0:
-                    print("not enough cash to buy long. cash",self.cash,"reserved_qty",reserved_qty,"reserved_price",reserved_price, "available",self.cash-reserved_price,"needed",float(int(size)*float(cena)))
+                    printanyway("not enough cash to buy long. cash",self.cash,"reserved_qty",reserved_qty,"reserved_price",reserved_price, "available",self.cash-reserved_price,"needed",float(int(size)*float(cena)))
                     return -1
 
         id = str(uuid4())
@@ -577,11 +577,11 @@ class Backtester:
         print("BT: replace order entry",id,size,price)
 
         if not price and not size:
-            print("size or price required")
+            printanyway("size or price required")
             return -1
 
         if len(self.open_orders) == 0:
-            print("BT: order doesnt exist")
+            printanyway("BT: order doesnt exist")
             return 0
         #with lock:
         for o in self.open_orders:
@@ -609,7 +609,7 @@ class Backtester:
         """
         print("BT: cancel order entry",id)
         if len(self.open_orders) == 0:
-            print("BTC: order doesnt exist")
+            printanyway("BTC: order doesnt exist")
             return 0
         #with lock:
         for o in self.open_orders:

v2realbot/common/model.py

@@ -94,12 +94,12 @@ class TestList(BaseModel):
 class Trade(BaseModel):
     symbol: str
     timestamp: datetime
-    exchange: Optional[Union[Exchange, str]]
+    exchange: Optional[Union[Exchange, str]] = None
     price: float
     size: float
     id: int
-    conditions: Optional[List[str]]
-    tape: Optional[str]
+    conditions: Optional[List[str]] = None
+    tape: Optional[str] = None
 
 
 #persisted object in pickle
@@ -114,8 +114,20 @@ class StrategyInstance(BaseModel):
     close_rush: int = 0
     stratvars_conf: str
     add_data_conf: str
-    note: Optional[str] 
-    history: Optional[str]
+    note: Optional[str]  = None
+    history: Optional[str] = None
+
+    def __setstate__(self, state: dict[Any, Any]) -> None:
+        """
+        Hack to allow unpickling models stored from pydantic V1
+        """
+        state.setdefault("__pydantic_extra__", {})
+        state.setdefault("__pydantic_private__", {})
+
+        if "__pydantic_fields_set__" not in state:
+            state["__pydantic_fields_set__"] = state.get("__fields_set__")
+
+        super().__setstate__(state)
 
 class RunRequest(BaseModel):
     id: UUID
@@ -125,8 +137,8 @@ class RunRequest(BaseModel):
     debug: bool = False
     strat_json: Optional[str] = None
     ilog_save: bool = False
-    bt_from: datetime = None
-    bt_to: datetime = None
+    bt_from: Optional[datetime] = None
+    bt_to: Optional[datetime] = None
     #weekdays filter
     #pokud je uvedeny filtrujeme tyto dny
     weekdays_filter: Optional[list] = None
@@ -147,8 +159,8 @@ class RunManagerRecord(BaseModel):
     mode: Mode
     note: Optional[str] = None
     ilog_save: bool = False
-    bt_from: datetime = None
-    bt_to: datetime = None
+    bt_from: Optional[datetime] = None
+    bt_to: Optional[datetime] = None
     #weekdays filter
     #pokud je uvedeny filtrujeme tyto dny
     weekdays_filter: Optional[list] = None #list of strings 0-6 representing days to run
@@ -156,9 +168,9 @@ class RunManagerRecord(BaseModel):
     batch_id: Optional[str] = None
     testlist_id: Optional[str] = None
     start_time: str #time (HH:MM) that start function is called
-    stop_time: Optional[str] #time  (HH:MM) that stop function is called
+    stop_time: Optional[str] = None #time  (HH:MM) that stop function is called
     status: SchedulerStatus
-    last_processed: Optional[datetime]
+    last_processed: Optional[datetime] = None
     history: Optional[str] = None
     valid_from: Optional[datetime] = None # US East time zone daetime
     valid_to: Optional[datetime] = None # US East time zone daetime
@@ -193,10 +205,10 @@ class Runner(BaseModel):
     run_name: Optional[str] = None
     run_note: Optional[str] = None
     run_ilog_save: Optional[bool] = False
-    run_trade_count: Optional[int]
-    run_profit: Optional[float]
-    run_positions: Optional[int]
-    run_avgp: Optional[float]
+    run_trade_count: Optional[int] = None
+    run_profit: Optional[float] = None
+    run_positions: Optional[int] = None
+    run_avgp: Optional[float] = None
     run_strat_json: Optional[str] = None
     run_stopped: Optional[datetime] = None
     run_paused: Optional[datetime] = None   
@@ -230,41 +242,41 @@ class Bar(BaseModel):
     low: float
     close: float
     volume: float
-    trade_count: Optional[float]
-    vwap: Optional[float]
+    trade_count: Optional[float] = 0
+    vwap: Optional[float] = 0
 
 class Order(BaseModel):
     id: UUID
     submitted_at: datetime
-    filled_at: Optional[datetime]
-    canceled_at: Optional[datetime]
+    filled_at: Optional[datetime] = None
+    canceled_at: Optional[datetime] = None
     symbol: str
     qty: int
     status: OrderStatus
     order_type: OrderType
-    filled_qty: Optional[int]
-    filled_avg_price: Optional[float]
+    filled_qty: Optional[int] = None
+    filled_avg_price: Optional[float] = None
     side: OrderSide
-    limit_price: Optional[float]
+    limit_price: Optional[float] = None
 
 #entita pro kazdy kompletni FILL, je navazana na prescribed_trade 
 class TradeUpdate(BaseModel):
     event: Union[TradeEvent, str]
-    execution_id: Optional[UUID]
+    execution_id: Optional[UUID] = None
     order: Order
     timestamp: datetime
-    position_qty: Optional[float]
-    price: Optional[float]
-    qty: Optional[float]
-    value: Optional[float]
-    cash: Optional[float]
-    pos_avg_price: Optional[float]
-    profit: Optional[float]
-    profit_sum: Optional[float]
-    rel_profit: Optional[float]
-    rel_profit_cum: Optional[float]
-    signal_name: Optional[str]
-    prescribed_trade_id: Optional[str]
+    position_qty: Optional[float] = None
+    price: Optional[float] = None
+    qty: Optional[float] = None
+    value: Optional[float] = None
+    cash: Optional[float] = None
+    pos_avg_price: Optional[float] = None
+    profit: Optional[float] = None
+    profit_sum: Optional[float] = None
+    rel_profit: Optional[float] = None
+    rel_profit_cum: Optional[float] = None
+    signal_name: Optional[str] = None
+    prescribed_trade_id: Optional[str] = None
 
 
 class RunArchiveChange(BaseModel):
@@ -289,8 +301,7 @@ class RunArchive(BaseModel):
     bt_from: Optional[datetime] = None
     bt_to: Optional[datetime] = None
     strat_json: Optional[str] = None
-    ##bude decomiss, misto toho stratvars_toml
-    stratvars: Optional[dict] = None
+    transferables: Optional[dict] = None #varaibles that are transferrable to next run
     settings: Optional[dict] = None
     ilog_save: Optional[bool] = False
     profit: float = 0
@@ -332,7 +343,7 @@ class RunArchiveViewPagination(BaseModel):
 
 #trida pro ukladani historie stoplossy do ext_data
 class SLHistory(BaseModel):
-    id: Optional[UUID]
+    id: Optional[UUID] = None
     time: datetime
     sl_val: float
 
@@ -345,7 +356,7 @@ class RunArchiveDetail(BaseModel):
     indicators: List[dict]
     statinds: dict
     trades: List[TradeUpdate]
-    ext_data: Optional[dict]
+    ext_data: Optional[dict] = None
     
 
 class InstantIndicator(BaseModel):

v2realbot/common/transform.py

@@ -83,5 +83,6 @@ def row_to_runarchive(row: dict) -> RunArchive:
         end_positions=int(row['end_positions']),
         end_positions_avgp=float(row['end_positions_avgp']),
         metrics=orjson.loads(row['metrics']),
-        stratvars_toml=row['stratvars_toml']
+        stratvars_toml=row['stratvars_toml'],
+        transferables=orjson.loads(row['transferables']) if row['transferables'] else None
     )

v2realbot/controller/configs.py

@@ -1,7 +1,6 @@
 
 import v2realbot.common.db as db
-from v2realbot.common.model import RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest
-import orjson
+from v2realbot.common.model import ConfigItem
 import v2realbot.utils.config_handler as ch
 
 # region CONFIG db services

v2realbot/controller/services.py

@@ -35,6 +35,7 @@ from sqlite3 import OperationalError, Row
 import v2realbot.strategyblocks.indicators.custom as ci
 from v2realbot.strategyblocks.inits.init_indicators import initialize_dynamic_indicators
 from v2realbot.strategyblocks.indicators.indicators_hub import populate_dynamic_indicators
+from v2realbot.strategyblocks.inits.init_attached_data import attach_previous_data
 from v2realbot.interfaces.backtest_interface import BacktestInterface
 import os
 import v2realbot.reporting.metricstoolsimage as mt
@@ -102,10 +103,10 @@ def create_stratin(si: StrategyInstance):
     #validate toml
     res, stp = parse_toml_string(si.stratvars_conf)
     if res < 0:
-        return (-1,"stratvars invalid")
+        return (-1,f"stratvars invalid: {stp}")
     res, adp = parse_toml_string(si.add_data_conf)
     if res < 0:
-        return (-1, "None")
+        return (-1, f"add data conf invalid {adp}") 
     si.id = uuid4()
     #print(si)
     db.stratins.append(si)
@@ -119,10 +120,10 @@ def modify_stratin(si: StrategyInstance, id: UUID):
         return (-1, "strat is running, use modify_stratin_running")
     res, stp = parse_toml_string(si.stratvars_conf)
     if res < 0:
-        return (-1, "stratvars invalid")
+        return (-1, f"stratvars invalid {stp}")
     res, adp = parse_toml_string(si.add_data_conf)
     if res < 0:
-        return (-1, "add data conf invalid") 
+        return (-1, f"add data conf invalid {adp}") 
     for i in db.stratins:
         if str(i.id) == str(id):
             #print("removing",i)
@@ -180,14 +181,14 @@ def modify_stratin_running(si: StrategyInstance, id: UUID):
         #validate toml
         res,stp = parse_toml_string(si.stratvars_conf)
         if res < 0:
-            return (-1, "new stratvars format invalid")
+            return (-1, f"new stratvars format invalid {stp}")
         for i in db.stratins:
             if str(i.id) == str(id):
                 if not is_stratin_running(id=str(id)):
                     return (-1, "not running")
                 res,stp_old = parse_toml_string(i.stratvars_conf)
                 if res < 0:
-                    return (-1, "current stratin stratvars invalid")
+                    return (-1, f"current stratin stratvars invalid {stp_old}")
                 #TODO reload running strat
                 #print(stp)
                 #print("starting injection", stp)
@@ -412,7 +413,7 @@ def run_batch_stratin(id: UUID, runReq: RunRequest):
     def get_market_days_in_interval(datefrom, dateto, note = None, id = None):
         #getting dates from calendat
         clientTrading = TradingClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, raw_data=False, paper=True)        
-        calendar_request = GetCalendarRequest(start=datefrom,end=dateto)
+        calendar_request = GetCalendarRequest(start=datefrom.date(),end=dateto.date())
         cal_dates = clientTrading.get_calendar(calendar_request)
         #list(Calendar)
         # Calendar
@@ -446,7 +447,7 @@ def run_batch_stratin(id: UUID, runReq: RunRequest):
             cal_list.append(RunDay(start = start_time, end = end_time, note = note, id = id))
 
         print(f"Getting interval dates from - to - RESULT ({len(cal_list)}):")
-        print(cal_list)
+        #print(cal_list)
         return cal_list
     
     #getting days to run into RunDays format
@@ -618,10 +619,10 @@ def run_stratin(id: UUID, runReq: RunRequest, synchronous: bool = False, inter_b
                 #validate toml
                 res, stp = parse_toml_string(i.stratvars_conf)
                 if res < 0:
-                    return (-1, "stratvars invalid")
+                    return (-1, f"stratvars invalid {stp}")
                 res, adp = parse_toml_string(i.add_data_conf)
                 if res < 0:
-                    return (-1, "add data conf invalid")
+                    return (-1, f"add data conf invalid {adp}")
                 id = uuid4()
                 print(f"RUN {id} INITIATED")
                 name = i.name
@@ -925,7 +926,8 @@ def archive_runner(runner: Runner, strat: StrategyInstance, inter_batch_params:
                                             end_positions=strat.state.positions,
                                             end_positions_avgp=round(float(strat.state.avgp),3),
                                             metrics=results_metrics,
-                                            stratvars_toml=runner.run_stratvars_toml
+                                            stratvars_toml=runner.run_stratvars_toml,
+                                            transferables=strat.state.vars["transferables"]
                                             )
         
         #flatten indicators from numpy array
@@ -1220,17 +1222,43 @@ def get_archived_runner_header_byID(id: UUID) -> RunArchive:
 #     else:
 #         return 0, res
 
-#vrátí seznam runneru s danym batch_id
-def get_archived_runnerslist_byBatchID(batch_id: str):
+# #vrátí seznam runneru s danym batch_id
+# def get_archived_runnerslist_byBatchID(batch_id: str):
+#     conn = pool.get_connection()
+#     try:
+#         cursor = conn.cursor()
+#         cursor.execute(f"SELECT runner_id FROM runner_header WHERE batch_id='{str(batch_id)}'")
+#         runner_list = [row[0] for row in cursor.fetchall()]
+#     finally:
+#         pool.release_connection(conn)
+#     return 0, runner_list
+
+#update that allows to sort
+def get_archived_runnerslist_byBatchID(batch_id: str, sort_order: str = "asc"):
+    """
+    Fetches all runner records by batch_id, sorted by the 'started' column.
+
+    :param batch_id: The batch ID to filter runners by.
+    :param sort_order: The sort order of the 'started' column. Defaults to 'asc'.
+                       Accepts 'asc' for ascending or 'desc' for descending order.
+    :return: A tuple with the first element being a status code and the second being the list of runner_ids.
+    """
+    # Validate sort_order
+    if sort_order.lower() not in ['asc', 'desc']:
+        return -1, []  # Returning an error code and an empty list in case of invalid sort_order
+    
     conn = pool.get_connection()
     try:
         cursor = conn.cursor()
-        cursor.execute(f"SELECT runner_id FROM runner_header WHERE batch_id='{str(batch_id)}'")
+        query = f"""SELECT runner_id FROM runner_header 
+                    WHERE batch_id=? 
+                    ORDER BY datetime(started) {sort_order.upper()}"""
+        cursor.execute(query, (batch_id,))
         runner_list = [row[0] for row in cursor.fetchall()]
     finally:
         pool.release_connection(conn)
     return 0, runner_list
-    
+
 def insert_archive_header(archeader: RunArchive):
     conn = pool.get_connection()
     try:
@@ -1239,11 +1267,11 @@ def insert_archive_header(archeader: RunArchive):
 
         res = c.execute("""
             INSERT INTO runner_header 
-            (runner_id, strat_id, batch_id, symbol, name, note, started, stopped, mode, account, bt_from, bt_to, strat_json, settings, ilog_save, profit, trade_count, end_positions, end_positions_avgp, metrics, stratvars_toml) 
+            (runner_id, strat_id, batch_id, symbol, name, note, started, stopped, mode, account, bt_from, bt_to, strat_json, settings, ilog_save, profit, trade_count, end_positions, end_positions_avgp, metrics, stratvars_toml, transferables) 
             VALUES
-            (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             """,
-            (str(archeader.id), str(archeader.strat_id), archeader.batch_id, archeader.symbol, archeader.name, archeader.note, archeader.started, archeader.stopped, archeader.mode, archeader.account, archeader.bt_from, archeader.bt_to, orjson.dumps(archeader.strat_json).decode('utf-8'), orjson.dumps(archeader.settings).decode('utf-8'), archeader.ilog_save, archeader.profit, archeader.trade_count, archeader.end_positions, archeader.end_positions_avgp, orjson.dumps(archeader.metrics, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME).decode('utf-8'), archeader.stratvars_toml))
+            (str(archeader.id), str(archeader.strat_id), archeader.batch_id, archeader.symbol, archeader.name, archeader.note, archeader.started, archeader.stopped, archeader.mode, archeader.account, archeader.bt_from, archeader.bt_to, orjson.dumps(archeader.strat_json).decode('utf-8'), orjson.dumps(archeader.settings).decode('utf-8'), archeader.ilog_save, archeader.profit, archeader.trade_count, archeader.end_positions, archeader.end_positions_avgp, orjson.dumps(archeader.metrics, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME).decode('utf-8'), archeader.stratvars_toml, orjson.dumps(archeader.transferables).decode('utf-8')))
 
         #retry not yet supported for statement format above
         #res = execute_with_retry(c,statement)
@@ -1567,7 +1595,7 @@ def preview_indicator_byTOML(id: UUID, indicator: InstantIndicator, save: bool =
         # print(row)
         res, toml_parsed = parse_toml_string(tomlino)
         if res < 0:
-            return (-2, "toml invalid")
+            return (-2, f"toml invalid: {toml_parsed}")
         
         #print("parsed toml", toml_parsed)
 
@@ -1664,10 +1692,15 @@ def preview_indicator_byTOML(id: UUID, indicator: InstantIndicator, save: bool =
 
         ##intialize required vars from strat init
         state.vars["loaded_models"] = {}
+        #state attributes for martingale sizing mngmt
+        state.vars["transferables"] = {}
+        state.vars["transferables"]["martingale"] = dict(cont_loss_series_cnt=0)
 
         ##intialize dynamic indicators
         initialize_dynamic_indicators(state)
-                    
+        #TODO vazit attached data (z toho potrebuji jen transferables, tzn. najit nejak predchozi runner a prelipnout transferables od zacatku)
+        #nejspis upravit attach_previous_data a nebo udelat specialni verzi
+        #attach_previous_data(state)     
 
         # print("subtype")   
         # function = "ci."+subtype+"."+subtype
@@ -1808,10 +1841,10 @@ def preview_indicator_byTOML(id: UUID, indicator: InstantIndicator, save: bool =
 
         #vracime list, kde pozice 0 je bar indicators, pozice 1 je ticks indicators
         if output == "bar":
-            return 0, [output_dict, []]
+            return 0, [output_dict, {}]
             #return 0, [new_inds[indicator.name], []]
         else:
-            return 0, [[], output_dict]
+            return 0, [{}, output_dict]
             #return 0, [[], new_tick_inds[indicator.name]]
 
     except Exception as e:
@@ -1896,7 +1929,7 @@ def get_alpaca_history_bars(symbol: str, datetime_object_from: datetime, datetim
         # Workaround of error when no data foun d AttributeError and has the specific message
         if isinstance(e, AttributeError) and str(e) == "'NoneType' object has no attribute 'items'":
             print("Caught the specific AttributeError: 'NoneType' object has no attribute 'items' means NO DATA FOUND")
-            #print(str(e) + format_exc())        
+            print(str(e) + format_exc())        
             return 0, result            
         else:
             print(str(e) + format_exc())

v2realbot/interfaces/live_interface.py

@@ -40,7 +40,9 @@ class LiveInterface(GeneralInterface):
             
             return market_order.id
         except Exception as e:
-            print("Nepodarilo se odeslat buy", str(e))
+            reason = "Nepodarilo se market buy:" + str(e) + format_exc()
+            print(reason)
+            send_to_telegram(reason)
             return -1
 
     """buy limit"""
@@ -65,7 +67,9 @@ class LiveInterface(GeneralInterface):
 
             return limit_order.id
         except Exception as e:
-            print("Nepodarilo se odeslat limitku", str(e))
+            reason = "Nepodarilo se odeslat buy limitku:" + str(e) + format_exc()
+            print(reason)
+            send_to_telegram(reason)
             return -1
 
     """sell market"""
@@ -87,7 +91,9 @@ class LiveInterface(GeneralInterface):
             
             return market_order.id
         except Exception as e:
-            print("Nepodarilo se odeslat sell", str(e))
+            reason = "Nepodarilo se odeslat sell:" + str(e) + format_exc()
+            print(reason)
+            send_to_telegram(reason)
             return -1
 
     """sell limit"""
@@ -112,8 +118,9 @@ class LiveInterface(GeneralInterface):
             return limit_order.id
         
         except Exception as e:
-            print("Nepodarilo se odeslat sell_l", str(e))
-            #raise Exception(e)
+            reason = "Nepodarilo se odeslat sell limitku:" + str(e) + format_exc()
+            print(reason)
+            send_to_telegram(reason)
             return -1
         
     """order replace"""
@@ -136,7 +143,9 @@ class LiveInterface(GeneralInterface):
             if e.code == 42210000: return orderid
             else:
                 ##mozna tady proste vracet vzdy ok
-                print("Neslo nahradit profitku. Problem",str(e))
+                reason = "Neslo nahradit profitku. Problem:" + str(e) + format_exc()
+                print(reason)
+                send_to_telegram(reason)
                 return -1
                 #raise Exception(e)
 
@@ -150,7 +159,9 @@ class LiveInterface(GeneralInterface):
             #order doesnt exist
             if e.code == 40410000: return 0
             else:
-                print("nepovedlo se zrusit objednavku", str(e))
+                reason = "Nepovedlo se zrusit objednavku:" + str(e) + format_exc()
+                print(reason)
+                send_to_telegram(reason)
                 #raise Exception(e)
                 return -1
             
@@ -178,7 +189,9 @@ class LiveInterface(GeneralInterface):
             #list of Orders (orderlist[0].id)
             return orderlist
         except Exception as e:
-            print("Chyba pri dotazeni objednávek.", str(e))
+            reason = "Chyba pri dotazeni objednávek:" + str(e) + format_exc()
+            print(reason)
+            send_to_telegram(reason)
             #raise Exception (e)
             return -1
     

v2realbot/main.py

@@ -690,7 +690,8 @@ def _generate_analysis(analyzerInputs: AnalyzerInputs):
 
         if res == 0: return StreamingResponse(stream, media_type="image/png")
         elif res < 0:
-            raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {res}:{id}")
+            print("Error when generating analysis: ",str(stream))
+            raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {res}:{stream}")
     except Exception as e:
             raise HTTPException(status_code=status.HTTP_406_NOT_ACCEPTABLE, detail=f"Error: {str(e)}" + format_exc())        
 

v2realbot/static/index.html

@@ -704,6 +704,10 @@
                                         <label for="stratvars" class="form-label">Stratvars</label>							
                                         <textarea class="form-control" rows="8" id="editstratvars" name="stratvars"></textarea>							
                                     </div>
+                                    <div class="form-group">
+                                        <label for="stratvars" class="form-label">Transferables</label>							
+                                        <textarea class="form-control" rows="8" id="edittransferables" name="stratvars"></textarea>							
+                                    </div>
                                     <div class="form-group">
                                         <label for="strat_json" class="form-label">Strat JSON</label>							
                                         <textarea class="form-control" rows="6" id="editstratjson" name="stratjson"></textarea>							
@@ -1164,7 +1168,7 @@
         <script src="/static/js/tables/archivetable/init.js?v=1.12"></script>
         <script src="/static/js/tables/archivetable/functions.js?v=1.10"></script>
         <script src="/static/js/tables/archivetable/modals.js?v=1.07"></script>
-        <script src="/static/js/tables/archivetable/handlers.js?v=1.08"></script>
+        <script src="/static/js/tables/archivetable/handlers.js?v=1.09"></script>
 
         <!-- Runmanager functionality -->
         <script src="/static/js/tables/runmanager/init.js?v=1.1"></script>

v2realbot/static/js/archivetables_firstbatchasheader.js

@@ -638,7 +638,7 @@ $(document).ready(function () {
             else{
                 $('#editstratvars').val(JSON.stringify(row.stratvars,null,2));
             }
-            
+            $('#edittransferables').val(JSON.stringify(row.transferables,null,2));
             
             $('#editstratjson').val(row.strat_json);
         }

v2realbot/static/js/tables/archivetable/functions.js

@@ -462,7 +462,7 @@ function display_batch_report(batch_id) {
 
 function refresh_logfile() {
     logfile = $("#logFileSelect").val()
-    lines = 700
+    lines = 1200
     $.ajax({
         url:"/log?lines="+lines+"&logfile="+logfile,
         beforeSend: function (xhr) {

v2realbot/static/js/tables/archivetable/handlers.js

@@ -445,7 +445,7 @@ $(document).ready(function () {
                 $('#editstratvars').val(JSON.stringify(row.stratvars,null,2));
             }
             
-            
+            $('#edittransferables').val(JSON.stringify(row.transferables,null,2));
             $('#editstratjson').val(row.strat_json);
         }
     });

v2realbot/strategy/StrategyClassicSL.py

@@ -9,7 +9,7 @@ from alpaca.trading.enums import TradeEvent, OrderStatus
 from v2realbot.indicators.indicators import ema
 import orjson
 from datetime import datetime
-#from rich import print
+from rich import print as printanyway
 from random import randrange
 from alpaca.common.exceptions import APIError
 import numpy as np
@@ -153,6 +153,10 @@ class StrategyClassicSL(Strategy):
                         self.state.rel_profit_cum.append(rel_profit)
                         rel_profit_cum_calculated = round(np.sum(self.state.rel_profit_cum),5)
 
+                        #pro martingale updatujeme loss_series_cnt
+                        self.state.vars["transferables"]["martingale"]["cont_loss_series_cnt"] = 0 if rel_profit > 0 else self.state.vars["transferables"]["martingale"]["cont_loss_series_cnt"]+1
+                        self.state.ilog(lvl=1, e=f"update cont_loss_series_cnt na {self.state.vars['transferables']['martingale']['cont_loss_series_cnt']}")
+
                 self.state.ilog(e=f"BUY notif - SHORT PROFIT: {partial_exit=} {partial_last=} {round(float(trade_profit),3)} celkem:{round(float(self.state.profit),3)} rel:{float(rel_profit)} rel_cum:{round(rel_profit_cum_calculated,7)}", msg=str(data.event), rel_profit_cum=str(self.state.rel_profit_cum), bought_amount=bought_amount, avg_costs=avg_costs, trade_qty=data.qty, trade_price=data.price, orderid=str(data.order.id))
 
                 #zapsat profit do prescr.trades
@@ -298,6 +302,10 @@ class StrategyClassicSL(Strategy):
                         self.state.rel_profit_cum.append(rel_profit)
                         rel_profit_cum_calculated = round(np.sum(self.state.rel_profit_cum),5)
 
+                        #pro martingale updatujeme loss_series_cnt
+                        self.state.vars["transferables"]["martingale"]["cont_loss_series_cnt"] = 0 if rel_profit > 0 else self.state.vars["transferables"]["martingale"]["cont_loss_series_cnt"]+1
+                        self.state.ilog(lvl=1, e=f"update cont_loss_series_cnt na {self.state.vars['transferables']['martingale']['cont_loss_series_cnt']}")
+
                 self.state.ilog(e=f"SELL notif - LONG PROFIT {partial_exit=} {partial_last=}:{round(float(trade_profit),3)} celkem:{round(float(self.state.profit),3)} rel:{float(rel_profit)} rel_cum:{round(rel_profit_cum_calculated,7)}", msg=str(data.event), rel_profit_cum = str(self.state.rel_profit_cum), sold_amount=sold_amount, avg_costs=avg_costs, trade_qty=data.qty, trade_price=data.price, orderid=str(data.order.id))
 
                 #zapsat profit do prescr.trades
@@ -423,12 +431,12 @@ class StrategyClassicSL(Strategy):
         #jde o uzavreni short pozice
         if int(self.state.positions) < 0 and (int(self.state.positions) + int(sizer)) > 0:
             self.state.ilog(e="buy nelze nakoupit vic nez shortuji", positions=self.state.positions, size=size)
-            print("buy nelze nakoupit vic nez shortuji") 
+            printanyway("buy nelze nakoupit vic nez shortuji") 
             return -2
 
         if int(self.state.positions) >= self.state.vars.maxpozic:
             self.state.ilog(e="buy Maxim mnozstvi naplneno", positions=self.state.positions)
-            print("max mnostvi naplneno")
+            printanyway("max mnostvi naplneno")
             return 0
 
         self.state.blockbuy = 1
@@ -447,13 +455,13 @@ class StrategyClassicSL(Strategy):
         #jde o uzavreni long pozice
         if int(self.state.positions) > 0 and (int(self.state.positions) - int(size)) < 0:
             self.state.ilog(e="nelze prodat vic nez longuji", positions=self.state.positions, size=size)
-            print("nelze prodat vic nez longuji") 
+            printanyway("nelze prodat vic nez longuji") 
             return -2
 
         #pokud shortuji a mam max pozic
         if int(self.state.positions) < 0 and abs(int(self.state.positions)) >= self.state.vars.maxpozic:
             self.state.ilog(e="short - Maxim mnozstvi naplneno", positions=self.state.positions, size=size)
-            print("max mnostvi naplneno")
+            printanyway("short - Maxim mnozstvi naplneno") 
             return 0
 
         #self.state.blocksell = 1

v2realbot/strategyblocks/inits/init_attached_data.py

@@ -0,0 +1,103 @@
+
+from v2realbot.common.model import RunDay, StrategyInstance, Runner, RunRequest, RunArchive, RunArchiveView, RunArchiveViewPagination, RunArchiveDetail, RunArchiveChange, Bar, TradeEvent, TestList, Intervals, ConfigItem, InstantIndicator, DataTablesRequest
+import v2realbot.controller.services as cs
+from v2realbot.utils.utils import slice_dict_lists,zoneUTC,safe_get, AttributeDict, filter_timeseries_by_timestamp
+#id = "b11c66d9-a9b6-475a-9ac1-28b11e1b4edf"
+#state = AttributeDict(vars={})
+from rich import print
+
+def attach_previous_data(state):
+    """""
+    Attaches data from previous runner of the same batch.
+    """""
+    print("ATTACHING PREVIOUS DATA")
+    runner : Runner
+    #get batch_id of current runer
+    res, runner = cs.get_runner(state.runner_id)
+    if res < 0:
+        if runner.batch_id is None:
+            print(f"No batch_id found for runner {runner.id}")
+        else:
+            print(f"Couldnt get previous runner {state.runner_id} error: {runner}")
+        return None
+    
+    batch_id = runner.batch_id
+    #batch_id = "6a6b0bcf"
+    res, runner_ids =cs.get_archived_runnerslist_byBatchID(batch_id, "desc")
+    if res < 0:
+        msg = f"error whne fetching runners of batch {batch_id} {runner_ids}"
+        print(msg)
+        return None
+    
+    if runner_ids is None or len(runner_ids) == 0:
+        print(f"NO runners found for batch {batch_id} {runner_ids}")
+        return None
+    
+    last_runner = runner_ids[0]
+    print("Previous runner identified:", last_runner)
+
+    #get archived header - to get transferables
+    runner_header : RunArchive = None
+    res, runner_header = cs.get_archived_runner_header_byID(last_runner)
+    if res < 0:
+        print(f"Error when fetching runner header {last_runner}")
+        return None
+
+    state.vars["transferables"] = runner_header.transferables
+    print("INITIALIZED transferables", state.vars["transferables"])
+
+
+    #get details from the runner
+    print(f"Fetching runner details of {last_runner}")
+    res, val = cs.get_archived_runner_details_byID(last_runner)
+    if res < 0:
+        print(f"no archived runner {last_runner}")
+        return None
+
+    detail = RunArchiveDetail(**val)
+    #print("toto jsme si dotahnuli", detail.bars)
+
+    # from stratvars directives 
+    attach_previous_bar_data = safe_get(state.vars, "attach_previous_bar_data", 50)
+    attach_previous_tick_data = safe_get(state.vars, "attach_previous_tick_data", None)
+    
+    #indicators datetime utc
+    indicators = slice_dict_lists(d=detail.indicators[0],last_item=attach_previous_bar_data, time_to_datetime=True)
+
+    #time -datetime utc, updated - timestamp float
+    bars = slice_dict_lists(d=detail.bars, last_item=attach_previous_bar_data, time_to_datetime=True)
+
+    #zarovname tick spolu s bar daty
+    if attach_previous_tick_data is None:
+        oldest_timestamp = bars["updated"][0]
+
+        #returns only values older that oldest_timestamp
+        cbar_inds = filter_timeseries_by_timestamp(detail.indicators[1], oldest_timestamp)
+    else:
+        cbar_inds = slice_dict_lists(d=detail.indicators[1],last_item=attach_previous_tick_data)
+
+    #USE these as INITs - TADY SI TO JESTE ZASTAVIT a POROVNAT
+    #print("state.indicatorsL", state.indicators, "NEW:", indicators)
+    state.indicators = AttributeDict(**indicators)
+    print("transfered indicators:", len(state.indicators["time"]))
+    #print("state.bars", state.bars, "NEW:", bars)
+    state.bars = AttributeDict(bars)
+    print("transfered bars:", len(state.bars["time"]))
+    #print("state.cbar_indicators", state.cbar_indicators, "NEW:", cbar_inds)
+    state.cbar_indicators = AttributeDict(cbar_inds)
+    print("transfered ticks:", len(state.cbar_indicators["time"]))
+
+    print("TRANSFERABLEs INITIALIZED")
+    #bars
+    #transferable_state_vars = ["martingale", "batch_profit"]
+    #1. pri initu se tyto klice v state vars se namapuji do ext_data ext_data["transferrables"]["martingale"] = state.vars["martingale"]
+    #2. pri transferu se vse z ext_data["trasferrables"] dá do stejnénné state.vars["martingale"]
+    #3. na konci dne se uloží do sloupce transferables v RunArchive
+
+    #pridavame dailyBars z extData
+    # if hasattr(detail, "ext_data") and "dailyBars" in detail.ext_data:
+    #     state.dailyBars = detail.ext_data["dailyBars"]
+    return
+
+# if __name__ == "__main__":
+#     attach_previous_data(state)

v2realbot/strategyblocks/newtrade/prescribedtrades.py

@@ -78,6 +78,7 @@ def execute_prescribed_trades(state: StrategyState, data):
                 size = state.vars.chunk
             res = state.sell(size=size)
             if isinstance(res, int) and res < 0:
+                print(f"error in required operation SHORT {res}")
                 raise Exception(f"error in required operation SHORT {res}")
             #defaultní goalprice nastavujeme az v notifikaci
 

v2realbot/strategyblocks/newtrade/sizing.py

@@ -147,7 +147,18 @@ def get_multiplier(state: StrategyState, data, signaloptions: dict, direction: T
         multiplier = f(input_value)
         state.ilog(lvl=1,e=f"SIZER - Interpolated value  {multiplier}", input_value=input_value, pattern_source_axis=pattern_source_axis, pattern_size_axis=pattern_size_axis, options=options, time=state.time)
     
-    if multiplier > 1 or multiplier <= 0:
+    martingale_enabled = utls.safe_get(options, "martingale_enabled", False)    
+
+    #pocet ztrátových obchodů v řadě mi udává multiplikátor (0 - 1, 1 ztráta 2x, 3 v řadě - 4x atp.)
+    if martingale_enabled:
+        cont_loss_series_cnt = state.vars["transferables"]["martingale"]["cont_loss_series_cnt"]
+        if cont_loss_series_cnt == 0:
+            multiplier = 1
+        else:
+            multiplier = 2 ** cont_loss_series_cnt
+        state.ilog(lvl=1,e=f"SIZER - MARTINGALE {multiplier}", options=options, time=state.time, cont_loss_series_cnt=cont_loss_series_cnt)
+
+    if (martingale_enabled is False and multiplier > 1) or multiplier <= 0:
         state.ilog(lvl=1,e=f"SIZER - Mame nekde problem MULTIPLIER mimo RANGE ERROR {multiplier}", options=options, time=state.time)
         multiplier = 1
     return multiplier

v2realbot/utils/utils.py

@@ -74,6 +74,16 @@ def fetch_calendar_data(start, end, max_retries=5, backoff_factor=1):
     :return: Calendar data.
     :raises: ConnectionError if all retries fail.
     """
+    # Ensure start and end are of type datetime.date
+    if isinstance(start, datetime):
+        start = start.date()
+    if isinstance(end, datetime):
+        end = end.date()
+
+    # Verify that start and end are datetime.date objects after conversion
+    if not all([isinstance(start, date), isinstance(end, date)]):
+        raise ValueError("start and end must be datetime.date objects")
+    
     clientTrading = TradingClient(ACCOUNT1_PAPER_API_KEY, ACCOUNT1_PAPER_SECRET_KEY, raw_data=False)
     calendar_request = GetCalendarRequest(start=start, end=end)
     last_exception = None
@@ -102,21 +112,45 @@ def concatenate_weekdays(weekday_filter):
     # Concatenate the weekday strings
     return ','.join(weekday_strings)
 
-def slice_dict_lists(d, last_item, to_tmstp = False):
+def filter_timeseries_by_timestamp(timeseries, timestamp):
+    """
+    Filter a timeseries dictionary, returning a new dictionary with entries
+    where the time value is greater than the provided timestamp.
+
+    Parameters:
+    - timeseries (dict): The original timeseries dictionary.
+    - timestamp (float): The timestamp to filter the timeseries by.
+
+    Returns:
+    - dict: A new timeseries dictionary filtered based on the provided timestamp.
+    """
+    # Find indices where time values are greater than the provided timestamp
+    indices = [i for i, time in enumerate(timeseries['time']) if time > timestamp]
+
+    # Create a new dictionary with values filtered by the indices
+    filtered_timeseries = {key: [value[i] for i in indices] for key, value in timeseries.items()}
+
+    return filtered_timeseries
+
+def slice_dict_lists(d, last_item, to_tmstp = False, time_to_datetime = False):
   """Slices every list in the dictionary to the last last_item items.
   
   Args:
     d: A dictionary.
     last_item: The number of items to keep at the end of each list.
-    to_tmstp: For "time" elements change it to timestamp from datetime if required.
-
+    to_tmstp: For "time" elements change it from datetime to timestamp from datetime if required.
+    time_to_datetime: For "time" elements change it from timestamp to datetime UTC if required.
   Returns:
     A new dictionary with the sliced lists.
+
+    datetime.fromtimestamp(data['updated']).astimezone(zoneUTC)
   """
   sliced_d = {}
   for key in d.keys():
     if key == "time" and to_tmstp:
         sliced_d[key] = [datetime.timestamp(t) for t in d[key][-last_item:]]
+    elif key == "time" and time_to_datetime:
+        sliced_d[key] = [datetime.fromtimestamp(t).astimezone(zoneUTC) for t in d[key][-last_item:]]
     else:
         sliced_d[key] = d[key][-last_item:]
   return sliced_d
@@ -644,8 +678,9 @@ def parse_toml_string(tomlst: str):
     try:
         tomlst = tomli.loads(tomlst)
     except tomli.TOMLDecodeError as e:
-        print("Not valid TOML.", str(e))
-        return (-1, None)
+        msg = f"Not valid TOML: " + str(e)
+        richprint(msg)
+        return (-1, msg)
     return (0, dict_replace_value(tomlst,"None",None))
 
 #class to persist