gui model metadata view + backend json optimalization orjson

v2realbot/utils/historicals.py

@@ -13,7 +13,7 @@ from collections import defaultdict
 from pandas import to_datetime
 from msgpack.ext import Timestamp
 
-def convert_daily_bars(daily_bars):
+def convert_historical_bars(daily_bars):
   """Converts a list of daily bars into a dictionary with the specified keys.
 
   Args:
@@ -89,4 +89,6 @@ def get_historical_bars(symbol: str, time_from: datetime, time_to: datetime, tim
     bar_request = StockBarsRequest(symbol_or_symbols=symbol,timeframe=timeframe, start=time_from, end=time_to, feed=DataFeed.SIP)
     bars: BarSet = stock_client.get_stock_bars(bar_request)
     #print("puvodni bars", bars["BAC"])
-    return convert_daily_bars(bars[symbol])
+    if bars[symbol][0] is None:
+       return None
+    return convert_historical_bars(bars[symbol])

v2realbot/utils/ilog.py

@@ -1,7 +1,7 @@
 from v2realbot.config import DATA_DIR
 from v2realbot.utils.utils import json_serial
 from uuid import UUID, uuid4
-import json
+import orjson
 from datetime import datetime
 from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account
 from v2realbot.common.db import pool, insert_queue
@@ -9,7 +9,7 @@ import sqlite3
 
 
 #standardne vraci pole tuplů, kde clen tuplu jsou sloupce
-#conn.row_factory = lambda c, r: json.loads(r[0])
+#conn.row_factory = lambda c, r: orjson.loads(r[0])
 #conn.row_factory = lambda c, r: r[0]
 #conn.row_factory = sqlite3.Row
 
@@ -32,7 +32,7 @@ def insert_log(runner_id: UUID, time: float, logdict: dict):
     conn = pool.get_connection()
     try:
         c = conn.cursor()
-        json_string = json.dumps(logdict, default=json_serial)
+        json_string = orjson.dumps(logdict, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME).decode('utf-8')
         res = c.execute("INSERT INTO runner_logs VALUES (?,?,?)",[str(runner_id), time, json_string])
         conn.commit()
     finally:
@@ -49,7 +49,7 @@ def insert_log_multiple_queue(runner_id:UUID, loglist: list):
 #     c = conn.cursor()
 #     insert_data = []
 #     for i in loglist:
-#         row = (str(runner_id), i["time"], json.dumps(i, default=json_serial))
+#         row = (str(runner_id), i["time"], orjson.dumps(i, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME))
 #         insert_data.append(row)
 #     c.executemany("INSERT INTO runner_logs VALUES (?,?,?)", insert_data)
 #     conn.commit()
@@ -59,7 +59,7 @@ def insert_log_multiple_queue(runner_id:UUID, loglist: list):
 def get_log_window(runner_id: UUID, timestamp_from: float = 0, timestamp_to: float = 9682851459):
     conn = pool.get_connection()
     try:
-        conn.row_factory = lambda c, r: json.loads(r[0])
+        conn.row_factory = lambda c, r: orjson.loads(r[0])
         c = conn.cursor()
         res = c.execute(f"SELECT data FROM runner_logs WHERE runner_id='{str(runner_id)}' AND time >={timestamp_from} AND time <={timestamp_to} ORDER BY time")
     finally:

v2realbot/utils/utils.py

@@ -329,6 +329,36 @@ def send_to_telegram(message):
     except Exception as e:
         print(e)
 
+def transform_data(data, transform_function):
+    """
+    Recursively transform the data in a dictionary, list of dictionaries, or nested dictionaries 
+    using a specified transformation function.
+
+    This function applies the transformation function to each value in the data structure. 
+    It handles nested dictionaries and lists of dictionaries.
+
+    Parameters:
+    data (dict or list): The dictionary, list of dictionaries, or nested dictionary to be transformed.
+    transform_function (function): The function to be applied to each value in the data. This function 
+                                   should accept a single value and return a transformed value.
+
+    Returns:
+    dict or list: The transformed dictionary, list of dictionaries, or nested dictionary with each value 
+                  processed by the transform_function.
+
+    Raises:
+    TypeError: If the transform_function cannot process a value, the original value is kept.
+    """
+    if isinstance(data, dict):
+        return {key: transform_data(value, transform_function) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [transform_data(element, transform_function) for element in data]
+    else:
+        try:
+            return transform_function(data)
+        except TypeError:
+            return data
+
 #OPTIMIZED BY BARD
 def json_serial(obj):
     """JSON serializer for objects not serializable by default json code
@@ -341,6 +371,7 @@ def json_serial(obj):
         UUID: lambda obj: str(obj),
         Enum: lambda obj: str(obj),
         np.int64: lambda obj: int(obj),
+        np.float64: lambda obj: float(obj),
         Order: lambda obj: obj.__dict__,
         TradeUpdate: lambda obj: obj.__dict__,
         btOrder: lambda obj: obj.__dict__,