From c29b1c553578f479d3fbe35e99666db85111dce9 Mon Sep 17 00:00:00 2001 From: David Brazda Date: Wed, 12 Jun 2024 13:33:50 +0200 Subject: [PATCH] all datetime columns changed to timestamp, change from json to faster orjson --- lightweight_charts/abstract.py | 12 ++++++++++ lightweight_charts/toolbox.py | 10 ++++----- lightweight_charts/util.py | 40 +++++++++++++++++++++++++++------- setup.py | 3 ++- 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/lightweight_charts/abstract.py b/lightweight_charts/abstract.py index 3ed7be3..b2a1b7e 100644 --- a/lightweight_charts/abstract.py +++ b/lightweight_charts/abstract.py @@ -275,6 +275,18 @@ class SeriesCommon(Pane): if not pd.api.types.is_datetime64_any_dtype(df['time']): df['time'] = pd.to_datetime(df['time']) df['time'] = df['time'].astype('int64') / 10 ** 9 #removed integer divison // 10 ** 9 to keep subseconds precision + + # if 'updated' in df.columns: + # df['updated'] = pd.to_datetime(df['updated']).astype('int64') / 10**9 + + # Iterate over all columns and convert any additional datetime columns to timestamps (for example updated) + for col in df.columns: + # Skip columns that are explicitly managed elsewhere or meant to be excluded + if col == 'time' or not pd.api.types.is_datetime64_any_dtype(df[col]): + continue + # Convert datetime to timestamp with nanosecond precision after the decimal + df[col] = pd.to_datetime(df[col]).astype('int64') / 10**9 + return df def _series_datetime_format(self, series: pd.Series, exclude_lowercase=None): diff --git a/lightweight_charts/toolbox.py b/lightweight_charts/toolbox.py index ac47045..998f33a 100644 --- a/lightweight_charts/toolbox.py +++ b/lightweight_charts/toolbox.py @@ -1,4 +1,4 @@ -import json +import orjson class ToolBox: @@ -22,14 +22,14 @@ class ToolBox: """ if not self.drawings.get(tag): return - self.run_script(f'if ({self.id}.toolBox) {self.id}.toolBox.loadDrawings({json.dumps(self.drawings[tag])})') + self.run_script(f'if ({self.id}.toolBox) {self.id}.toolBox.loadDrawings({orjson.dumps(self.drawings[tag])})') def import_drawings(self, file_path): """ Imports a list of drawings stored at the given file path. """ with open(file_path, 'r') as f: - json_data = json.load(f) + json_data = orjson.load(f) self.drawings = json_data def export_drawings(self, file_path): @@ -37,9 +37,9 @@ class ToolBox: Exports the current list of drawings to the given file path. """ with open(file_path, 'w+') as f: - json.dump(self.drawings, f, indent=4) + orjson.dump(self.drawings, f, indent=4) def _save_drawings(self, drawings): if not self._save_under: return - self.drawings[self._save_under.value] = json.loads(drawings) + self.drawings[self._save_under.value] = orjson.loads(drawings) diff --git a/lightweight_charts/util.py b/lightweight_charts/util.py index 33483a8..9933e52 100644 --- a/lightweight_charts/util.py +++ b/lightweight_charts/util.py @@ -1,5 +1,5 @@ import asyncio -import json +import orjson from datetime import datetime from random import choices from typing import Literal, Union @@ -36,30 +36,54 @@ def parse_event_message(window, string): return func, args +# def js_data(data: Union[pd.DataFrame, pd.Series]): +# if isinstance(data, pd.DataFrame): +# d = data.to_dict(orient='records') +# filtered_records = [{k: v for k, v in record.items() if v is not None and not pd.isna(v)} for record in d] +# else: +# d = data.to_dict() +# filtered_records = {k: v for k, v in d.items()} +# return json.dumps(filtered_records) + def js_data(data: Union[pd.DataFrame, pd.Series]): if isinstance(data, pd.DataFrame): - d = data.to_dict(orient='records') - filtered_records = [{k: v for k, v in record.items() if v is not None and not pd.isna(v)} for record in d] + # Converting DataFrame to a list of dictionaries, filtering out NaN values + filtered_records = data.dropna().to_dict(orient='records') else: - d = data.to_dict() - filtered_records = {k: v for k, v in d.items()} - return json.dumps(filtered_records, indent=2) + # For pd.Series, convert to dict and drop NaN values + filtered_records = data.dropna().to_dict() + # Serialize using orjson, which returns bytes + # Decode bytes to string if necessary (JavaScript consumption requires string) + return orjson.dumps(filtered_records).decode('utf-8') def snake_to_camel(s: str): components = s.split('_') return components[0] + ''.join(x.title() for x in components[1:]) +# def js_json(d: dict): +# filtered_dict = {} +# for key, val in d.items(): +# if key in ('self') or val in (None,): +# continue +# if '_' in key: +# key = snake_to_camel(key) +# filtered_dict[key] = val +# return f"JSON.parse('{json.dumps(filtered_dict)}')" + def js_json(d: dict): filtered_dict = {} for key, val in d.items(): - if key in ('self') or val in (None,): + if key == 'self' or val in (None,): continue if '_' in key: key = snake_to_camel(key) filtered_dict[key] = val - return f"JSON.parse('{json.dumps(filtered_dict)}')" + # Serialize the dictionary using orjson, automatically handling types that orjson can serialize + # Decode the bytes to string for use in JavaScript, escaping single quotes for JavaScript consumption + json_str = orjson.dumps(filtered_dict).decode('utf-8').replace("'", "\\'") + return f"JSON.parse('{json_str}')" def jbool(b: bool): return 'true' if b is True else 'false' if b is False else None diff --git a/setup.py b/setup.py index eef3fa2..fd20152 100644 --- a/setup.py +++ b/setup.py @@ -5,12 +5,13 @@ with open('README.md', 'r', encoding='utf-8') as f: setup( name='lightweight_charts', - version='2.0.15', + version='2.0.16', packages=find_packages(), python_requires='>=3.8', install_requires=[ 'pandas', 'pywebview>=5.0.5', + 'orjson' ], package_data={ 'lightweight_charts': ['js/*'],