dwker/v2realbot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gui model metadata view + backend json optimalization orjson

Browse Source
This commit is contained in:
David Brazda
2023-12-10 15:02:25 +01:00
parent 523905ece6
commit 232f32467e
38 changed files with 224 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
testy/archive/alpacasnapshot.py
View File

@ -23,7 +23,7 @@ from rich import print
from collections import defaultdict
from pandas import to_datetime
from msgpack.ext import Timestamp
from v2realbot.utils.historicals import convert_daily_bars
from v2realbot.utils.historicals import convert_historical_bars
def get_last_close():
pass
@ -38,7 +38,7 @@ def get_historical_bars(symbol: str, time_from: datetime, time_to: datetime, tim
bars: BarSet = stock_client.get_stock_bars(bar_request)
print("puvodni bars", bars["BAC"])
print(bars)
return convert_daily_bars(bars[symbol])
return convert_historical_bars(bars[symbol])
#v initu plnime pozadovana historicka data do historicals[]

4
testy/ascending.py
View File

File diff suppressed because one or more lines are too long

10
testy/migrace/migracerunnerheader.py
View File

@ -2,7 +2,7 @@ import sqlite3
from v2realbot.config import DATA_DIR
from v2realbot.utils.utils import json_serial
from uuid import UUID, uuid4
import json
import orjson
from datetime import datetime
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account
from v2realbot.common.model import RunArchiveDetail, RunArchive, RunArchiveView
@ -35,14 +35,14 @@ def row_to_object(row: dict) -> RunArchive:
end_positions=row.get('end_positions'),
end_positions_avgp=row.get('end_positions_avgp'),
metrics=row.get('open_orders'),
#metrics=json.loads(row.get('metrics')) if row.get('metrics') else None,
#metrics=orjson.loads(row.get('metrics')) if row.get('metrics') else None,
stratvars_toml=row.get('stratvars_toml')
)
def get_all_archived_runners():
conn = pool.get_connection()
try:
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_header")
finally:
@ -54,7 +54,7 @@ def insert_archive_header(archeader: RunArchive):
conn = pool.get_connection()
try:
c = conn.cursor()
json_string = json.dumps(archeader, default=json_serial)
json_string = orjson.dumps(archeader, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
if archeader.batch_id is not None:
statement = f"INSERT INTO runner_header (runner_id, batch_id, ra) VALUES ('{str(archeader.id)}','{str(archeader.batch_id)}','{json_string}')"
else:
@ -103,7 +103,7 @@ def migrate_to_columns(ra: RunArchive):
SET strat_id=?, batch_id=?, symbol=?, name=?, note=?, started=?, stopped=?, mode=?, account=?, bt_from=?, bt_to=?, strat_json=?, settings=?, ilog_save=?, profit=?, trade_count=?, end_positions=?, end_positions_avgp=?, metrics=?, stratvars_toml=?
WHERE runner_id=?
''',
(str(ra.strat_id), ra.batch_id, ra.symbol, ra.name, ra.note, ra.started, ra.stopped, ra.mode, ra.account, ra.bt_from, ra.bt_to, json.dumps(ra.strat_json), json.dumps(ra.settings), ra.ilog_save, ra.profit, ra.trade_count, ra.end_positions, ra.end_positions_avgp, json.dumps(ra.metrics), ra.stratvars_toml, str(ra.id)))
(str(ra.strat_id), ra.batch_id, ra.symbol, ra.name, ra.note, ra.started, ra.stopped, ra.mode, ra.account, ra.bt_from, ra.bt_to, orjson.dumps(ra.strat_json), orjson.dumps(ra.settings), ra.ilog_save, ra.profit, ra.trade_count, ra.end_positions, ra.end_positions_avgp, orjson.dumps(ra.metrics), ra.stratvars_toml, str(ra.id)))
conn.commit()
finally:

20
testy/migrace/migracesql.py
View File

@ -2,7 +2,7 @@ import sqlite3
from v2realbot.config import DATA_DIR
from v2realbot.utils.utils import json_serial
from uuid import UUID, uuid4
import json
import orjson
from datetime import datetime
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account
from v2realbot.common.model import RunArchiveDetail
@ -11,7 +11,7 @@ from tinydb import TinyDB, Query, where
sqlite_db_file = DATA_DIR + "/v2trading.db"
conn = sqlite3.connect(sqlite_db_file)
#standardne vraci pole tuplů, kde clen tuplu jsou sloupce
#conn.row_factory = lambda c, r: json.loads(r[0])
#conn.row_factory = lambda c, r: orjson.loads(r[0])
#conn.row_factory = lambda c, r: r[0]
#conn.row_factory = sqlite3.Row
@ -28,7 +28,7 @@ insert_list = [dict(time=datetime.now().timestamp(), side="ddd", rectype=RecordT
def insert_log(runner_id: UUID, time: float, logdict: dict):
c = conn.cursor()
json_string = json.dumps(logdict, default=json_serial)
json_string = orjson.dumps(logdict, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
res = c.execute("INSERT INTO runner_logs VALUES (?,?,?)",[str(runner_id), time, json_string])
conn.commit()
return res.rowcount
@ -37,14 +37,14 @@ def insert_log_multiple(runner_id: UUID, loglist: list):
c = conn.cursor()
insert_data = []
for i in loglist:
row = (str(runner_id), i["time"], json.dumps(i, default=json_serial))
row = (str(runner_id), i["time"], orjson.dumps(i, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME))
insert_data.append(row)
c.executemany("INSERT INTO runner_logs VALUES (?,?,?)", insert_data)
conn.commit()
return c.rowcount
# c = conn.cursor()
# json_string = json.dumps(logdict, default=json_serial)
# json_string = orjson.dumps(logdict, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
# res = c.execute("INSERT INTO runner_logs VALUES (?,?,?)",[str(runner_id), time, json_string])
# print(res)
# conn.commit()
@ -52,7 +52,7 @@ def insert_log_multiple(runner_id: UUID, loglist: list):
#returns list of ilog jsons
def read_log_window(runner_id: UUID, timestamp_from: float, timestamp_to: float):
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_logs WHERE runner_id='{str(runner_id)}' AND time >={ts_from} AND time <={ts_to}")
return res.fetchall()
@ -94,21 +94,21 @@ def delete_logs(runner_id: UUID):
def insert_archive_detail(archdetail: RunArchiveDetail):
c = conn.cursor()
json_string = json.dumps(archdetail, default=json_serial)
json_string = orjson.dumps(archdetail, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
res = c.execute("INSERT INTO runner_detail VALUES (?,?)",[str(archdetail["id"]), json_string])
conn.commit()
return res.rowcount
#returns list of details
def get_all_archive_detail():
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_detail")
return res.fetchall()
#vrátí konkrétní
def get_archive_detail_byID(runner_id: UUID):
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_detail WHERE runner_id='{str(runner_id)}'")
return res.fetchone()
@ -123,7 +123,7 @@ def delete_archive_detail(runner_id: UUID):
def get_all_archived_runners_detail():
arch_detail_file = DATA_DIR + "/arch_detail.json"
db_arch_d = TinyDB(arch_detail_file, default=json_serial)
db_arch_d = TinyDB(arch_detail_file, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
res = db_arch_d.all()
return 0, res

2
testy/ml/test.py
View File

@ -4,7 +4,7 @@ from keras.models import Sequential
from keras.layers import LSTM, Dense
from v2realbot.controller.services import get_archived_runner_details_byID
from v2realbot.common.model import RunArchiveDetail
import json
import orjson
runner_id = "838e918e-9be0-4251-a968-c13c83f3f173"
result = None

20
testy/testSqlite3.py
View File

@ -2,7 +2,7 @@ import sqlite3
from v2realbot.config import DATA_DIR
from v2realbot.utils.utils import json_serial
from uuid import UUID, uuid4
import json
import orjson
from datetime import datetime
from v2realbot.enums.enums import RecordType, StartBarAlign, Mode, Account
from v2realbot.common.model import RunArchiveDetail
@ -11,7 +11,7 @@ from tinydb import TinyDB, Query, where
sqlite_db_file = DATA_DIR + "/v2trading.db"
conn = sqlite3.connect(sqlite_db_file)
#standardne vraci pole tuplů, kde clen tuplu jsou sloupce
#conn.row_factory = lambda c, r: json.loads(r[0])
#conn.row_factory = lambda c, r: orjson.loads(r[0])
#conn.row_factory = lambda c, r: r[0]
#conn.row_factory = sqlite3.Row
@ -28,7 +28,7 @@ insert_list = [dict(time=datetime.now().timestamp(), side="ddd", rectype=RecordT
def insert_log(runner_id: UUID, time: float, logdict: dict):
c = conn.cursor()
json_string = json.dumps(logdict, default=json_serial)
json_string = orjson.dumps(logdict, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
res = c.execute("INSERT INTO runner_logs VALUES (?,?,?)",[str(runner_id), time, json_string])
conn.commit()
return res.rowcount
@ -37,14 +37,14 @@ def insert_log_multiple(runner_id: UUID, loglist: list):
c = conn.cursor()
insert_data = []
for i in loglist:
row = (str(runner_id), i["time"], json.dumps(i, default=json_serial))
row = (str(runner_id), i["time"], orjson.dumps(i, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME))
insert_data.append(row)
c.executemany("INSERT INTO runner_logs VALUES (?,?,?)", insert_data)
conn.commit()
return c.rowcount
# c = conn.cursor()
# json_string = json.dumps(logdict, default=json_serial)
# json_string = orjson.dumps(logdict, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
# res = c.execute("INSERT INTO runner_logs VALUES (?,?,?)",[str(runner_id), time, json_string])
# print(res)
# conn.commit()
@ -52,7 +52,7 @@ def insert_log_multiple(runner_id: UUID, loglist: list):
#returns list of ilog jsons
def read_log_window(runner_id: UUID, timestamp_from: float, timestamp_to: float):
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_logs WHERE runner_id='{str(runner_id)}' AND time >={ts_from} AND time <={ts_to}")
return res.fetchall()
@ -94,21 +94,21 @@ def delete_logs(runner_id: UUID):
def insert_archive_detail(archdetail: RunArchiveDetail):
c = conn.cursor()
json_string = json.dumps(archdetail, default=json_serial)
json_string = orjson.dumps(archdetail, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
res = c.execute("INSERT INTO runner_detail VALUES (?,?)",[str(archdetail["id"]), json_string])
conn.commit()
return res.rowcount
#returns list of details
def get_all_archive_detail():
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_detail")
return res.fetchall()
#vrátí konkrétní
def get_archive_detail_byID(runner_id: UUID):
conn.row_factory = lambda c, r: json.loads(r[0])
conn.row_factory = lambda c, r: orjson.loads(r[0])
c = conn.cursor()
res = c.execute(f"SELECT data FROM runner_detail WHERE runner_id='{str(runner_id)}'")
return res.fetchone()
@ -123,7 +123,7 @@ def delete_archive_detail(runner_id: UUID):
def get_all_archived_runners_detail():
arch_detail_file = DATA_DIR + "/arch_detail.json"
db_arch_d = TinyDB(arch_detail_file, default=json_serial)
db_arch_d = TinyDB(arch_detail_file, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
res = db_arch_d.all()
return 0, res

2
testy/testStore.py
View File

@ -46,7 +46,7 @@ db.save()
# b = 2
# def toJson(self):
# return json.dumps(self, default=lambda o: o.__dict__)
# return orjson.dumps(self, default=lambda o: o.__dict__)
# db.append(Neco.a)

4
testy/testTIMIT
View File

@ -1,12 +1,12 @@
import timeit
setup = '''
import msgpack
import json
import orjson
from copy import deepcopy
data = {'name':'John Doe','ranks':{'sports':13,'edu':34,'arts':45},'grade':5}'''
print(timeit.timeit('deepcopy(data)', setup=setup))
# 12.0860249996
print(timeit.timeit('json.loads(json.dumps(data))', setup=setup))
print(timeit.timeit('orjson.loads(orjson.dumps(data))', setup=setup))
# 9.07182312012
print(timeit.timeit('msgpack.unpackb(msgpack.packb(data))', setup=setup))
# 1.42743492126

4
testy/tinyDBselect.py
View File

@ -16,7 +16,7 @@ import importlib
from queue import Queue
from tinydb import TinyDB, Query, where
from tinydb.operations import set
import json
import orjson
from rich import print
@ -29,7 +29,7 @@ class RunnerLogger:
def __init__(self, runner_id: UUID) -> None:
self.runner_id = runner_id
runner_log_file = DATA_DIR + "/runner_log.json"
db_runner_log = TinyDB(runner_log_file, default=json_serial)
db_runner_log = TinyDB(runner_log_file, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
def insert_log_multiple(runner_id: UUID, logList: list):
runner_table = db_runner_log.table(str(runner_id))

4
testy/tinyFLUXtest.py
View File

@ -16,7 +16,7 @@ import importlib
from queue import Queue
#from tinydb import TinyDB, Query, where
#from tinydb.operations import set
import json
import orjson
from rich import print
from tinyflux import Point, TinyFlux
@ -26,7 +26,7 @@ runner_log_file = DATA_DIR + "/runner_flux__log.json"
db_runner_log = TinyFlux(runner_log_file)
insert_dict = {'datum': datetime.now(), 'side': "dd", 'name': 'david','id': uuid4(), 'order': "neco"}
#json.dumps(insert_dict, default=json_serial)
#orjson.dumps(insert_dict, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
p1 = Point(time=datetime.now(), tags=insert_dict)
db_runner_log.insert(p1)

4
testy/tinydbTest.py
View File

@ -13,7 +13,7 @@ from v2realbot.common.model import Order, TradeUpdate as btTradeUpdate
from alpaca.trading.models import TradeUpdate
from alpaca.trading.enums import TradeEvent, OrderType, OrderSide, OrderType, OrderStatus
from rich import print
import json
import orjson
#storage_with_injected_serialization = JSONStorage()
@ -110,7 +110,7 @@ a = Order(id=uuid4(),
limit_price=22.4)
db_file = DATA_DIR + "/db.json"
db = TinyDB(db_file, default=json_serial)
db = TinyDB(db_file, default=json_serial, option=orjson.OPT_PASSTHROUGH_DATETIME)
db.truncate()
insert = {'datum': datetime.now(), 'side': OrderSide.BUY, 'name': 'david','id': uuid4(), 'order': orderList}

4
testy/websocketFastApi.py
View File

@ -6,7 +6,7 @@ import secrets
from typing import Annotated
import os
import uvicorn
import json
import orjson
from datetime import datetime
from v2realbot.utils.utils import zoneNY
@ -103,7 +103,7 @@ async def websocket_endpoint(
'vwap': 123,
'updated': 123,
'index': 123}
await websocket.send_text(json.dumps(data))
await websocket.send_text(orjson.dumps(data))
except WebSocketDisconnect:
print("CLIENT DISCONNECTED for", runner_id)

4
testy/websocketFastApiConnManager.py
View File

@ -6,7 +6,7 @@ import secrets
from typing import Annotated
import os
import uvicorn
import json
import orjson
from datetime import datetime
from v2realbot.utils.utils import zoneNY
@ -101,7 +101,7 @@ async def websocket_endpoint(websocket: WebSocket, client_id: int):
# 'close': 123,
# 'open': 123,
# 'time': "2019-05-25"}
await manager.send_personal_message(json.dumps(data), websocket)
await manager.send_personal_message(orjson.dumps(data), websocket)
#await manager.broadcast(f"Client #{client_id} says: {data}")
except WebSocketDisconnect:
manager.disconnect(websocket)