all datetime columns changed to timestamp, change from json to faster orjson

lightweight_charts/abstract.py

@@ -275,6 +275,18 @@ class SeriesCommon(Pane):
         if not pd.api.types.is_datetime64_any_dtype(df['time']):
             df['time'] = pd.to_datetime(df['time'])
         df['time'] = df['time'].astype('int64') / 10 ** 9 #removed integer divison // 10 ** 9 to keep subseconds precision
+
+        # if 'updated' in df.columns:
+        #     df['updated'] = pd.to_datetime(df['updated']).astype('int64') / 10**9
+
+        # Iterate over all columns and convert any additional datetime columns to timestamps (for example updated)
+        for col in df.columns:
+            # Skip columns that are explicitly managed elsewhere or meant to be excluded
+            if col == 'time' or not pd.api.types.is_datetime64_any_dtype(df[col]):
+                continue
+            # Convert datetime to timestamp with nanosecond precision after the decimal
+            df[col] = pd.to_datetime(df[col]).astype('int64') / 10**9
+
         return df
 
     def _series_datetime_format(self, series: pd.Series, exclude_lowercase=None):

lightweight_charts/toolbox.py

@@ -1,4 +1,4 @@
-import json
+import orjson
 
 
 class ToolBox:
@@ -22,14 +22,14 @@ class ToolBox:
         """
         if not self.drawings.get(tag):
             return
-        self.run_script(f'if ({self.id}.toolBox) {self.id}.toolBox.loadDrawings({json.dumps(self.drawings[tag])})')
+        self.run_script(f'if ({self.id}.toolBox) {self.id}.toolBox.loadDrawings({orjson.dumps(self.drawings[tag])})')
 
     def import_drawings(self, file_path):
         """
         Imports a list of drawings stored at the given file path.
         """
         with open(file_path, 'r') as f:
-            json_data = json.load(f)
+            json_data = orjson.load(f)
             self.drawings = json_data
 
     def export_drawings(self, file_path):
@@ -37,9 +37,9 @@ class ToolBox:
         Exports the current list of drawings to the given file path.
         """
         with open(file_path, 'w+') as f:
-            json.dump(self.drawings, f, indent=4)
+            orjson.dump(self.drawings, f, indent=4)
 
     def _save_drawings(self, drawings):
         if not self._save_under:
             return
-        self.drawings[self._save_under.value] = json.loads(drawings)
+        self.drawings[self._save_under.value] = orjson.loads(drawings)

lightweight_charts/util.py

@@ -1,5 +1,5 @@
 import asyncio
-import json
+import orjson
 from datetime import datetime
 from random import choices
 from typing import Literal, Union
@@ -36,30 +36,54 @@ def parse_event_message(window, string):
     return func, args
 
 
+# def js_data(data: Union[pd.DataFrame, pd.Series]):
+#     if isinstance(data, pd.DataFrame):
+#         d = data.to_dict(orient='records')
+#         filtered_records = [{k: v for k, v in record.items() if v is not None and not pd.isna(v)} for record in d]
+#     else:
+#         d = data.to_dict()
+#         filtered_records = {k: v for k, v in d.items()}
+#     return json.dumps(filtered_records)
+
 def js_data(data: Union[pd.DataFrame, pd.Series]):
     if isinstance(data, pd.DataFrame):
-        d = data.to_dict(orient='records')
-        filtered_records = [{k: v for k, v in record.items() if v is not None and not pd.isna(v)} for record in d]
+        # Converting DataFrame to a list of dictionaries, filtering out NaN values
+        filtered_records = data.dropna().to_dict(orient='records')
     else:
-        d = data.to_dict()
-        filtered_records = {k: v for k, v in d.items()}
-    return json.dumps(filtered_records, indent=2)
+        # For pd.Series, convert to dict and drop NaN values
+        filtered_records = data.dropna().to_dict()
 
+    # Serialize using orjson, which returns bytes
+    # Decode bytes to string if necessary (JavaScript consumption requires string)
+    return orjson.dumps(filtered_records).decode('utf-8')
 
 def snake_to_camel(s: str):
     components = s.split('_')
     return components[0] + ''.join(x.title() for x in components[1:])
 
+# def js_json(d: dict):
+#     filtered_dict = {}
+#     for key, val in d.items():
+#         if key in ('self') or val in (None,):
+#             continue
+#         if '_' in key:
+#             key = snake_to_camel(key)
+#         filtered_dict[key] = val
+#     return f"JSON.parse('{json.dumps(filtered_dict)}')"
+
 def js_json(d: dict):
     filtered_dict = {}
     for key, val in d.items():
-        if key in ('self') or val in (None,):
+        if key == 'self' or val in (None,):
             continue
         if '_' in key:
             key = snake_to_camel(key)
         filtered_dict[key] = val
-    return f"JSON.parse('{json.dumps(filtered_dict)}')"
 
+    # Serialize the dictionary using orjson, automatically handling types that orjson can serialize
+    # Decode the bytes to string for use in JavaScript, escaping single quotes for JavaScript consumption
+    json_str = orjson.dumps(filtered_dict).decode('utf-8').replace("'", "\\'")
+    return f"JSON.parse('{json_str}')"
 
 def jbool(b: bool): return 'true' if b is True else 'false' if b is False else None