dwker/strategy-lab
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

daily update

Browse Source
This commit is contained in:
David Brazda
2024-10-21 20:57:56 +02:00
parent 132172855a
commit e3da60c647
196 changed files with 1722489 additions and 1134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
research/data/db_loader_saver.ipynb
View File

@ -9,11 +9,33 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 3,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"None\n",
"Loaded env variables from file None\n"
]
},
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Activating profile profile1\n",
"</pre>\n"
],
"text/plain": [
"Activating profile profile1\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from lightweight_charts import Panel, chart, PlotAccessor\n",
"from lightweight_charts import Panel, chart, PlotDFAccessor\n",
"from v2realbot.utils.utils import zoneNY\n",
"import pandas as pd\n",
"import numpy as np\n",
@ -496,7 +518,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 4,
"metadata": {},
"outputs": [
{
@ -508,25 +530,34 @@
"trades_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n",
"trades_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
"ohlcv_df-BAC-2024-01-11T09:30:00-2024-01-12T16:00:00.parquet\n",
"trades_df-BAC-2024-10-03T09:30:00-2024-10-16T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
"trades_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
"trades_df-BAC-2024-05-15T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
"ohlcv_df-BAC-2024-10-03T09:30:00-2024-10-16T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
"ohlcv_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\n",
"ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
"ohlcv_df-SPY-2024-01-01T09:30:00-2024-05-14T16:00:00.parquet\n",
"ohlcv_df-BAC-2024-01-01T09_30_00-2024-05-14T16_00_00-CO4B7VPWUZF-100.parquet\n",
"ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T16_00_00-47BCFOPUVWZ-100.parquet\n",
"ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T15_30_00-47BCFOPUVWZ-100.parquet\n"
"ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T15_30_00-47BCFOPUVWZ-100.parquet\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"DatetimeIndex: 114097 entries, 2024-10-03 09:30:00-04:00 to 2024-10-16 15:59:59-04:00\n",
"Data columns (total 10 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 open 114097 non-null float64 \n",
" 1 high 114097 non-null float64 \n",
" 2 low 114097 non-null float64 \n",
" 3 close 114097 non-null float64 \n",
" 4 volume 114097 non-null float64 \n",
" 5 trades 114097 non-null float64 \n",
" 6 updated 114097 non-null datetime64[ns, US/Eastern]\n",
" 7 vwap 114097 non-null float64 \n",
" 8 buyvolume 114097 non-null float64 \n",
" 9 sellvolume 114097 non-null float64 \n",
"dtypes: datetime64[ns, US/Eastern](1), float64(9)\n",
"memory usage: 9.6 MB\n"
]
},
{
"data": {
"text/plain": [
"440"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
@ -545,7 +576,8 @@
"files = [f for f in os.listdir(dir) if f.endswith(\".parquet\")]\n",
"print('\\n'.join(map(str, files)))\n",
"#file_name = \"ohlcv_df-BAC-2023-01-01T09_30_00-2024-05-25T15_30_00-47BCFOPUVWZ-100.parquet\"\n",
"file_name = \"ohlcv_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\"\n",
"#file_name = \"ohlcv_df-BAC-2023-01-01T09:30:00-2024-10-02T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\"\n",
"file_name = \"ohlcv_df-BAC-2024-10-03T09:30:00-2024-10-16T16:00:00-['4', '7', 'B', 'C', 'F', 'O', 'P', 'U', 'V', 'W', 'Z']-100.parquet\"\n",
"ohlcv_df = pd.read_parquet(dir+file_name,engine='pyarrow')\n",
"#filter ohlcv_df to certain date range (assuming datetime index)\n",
"#ohlcv_df = ohlcv_df.loc[\"2024-02-12 9:30\":\"2024-02-16 16:00\"]\n",
@ -555,9 +587,29 @@
"\n",
"basic_data = vbt.Data.from_data(vbt.symbol_dict({\"BAC\": ohlcv_df}), tz_convert=zoneNY)\n",
"#ohlcv_df= None\n",
"basic_data.wrapper.index.normalize().nunique()\n",
"bd = basic_data.transform(lambda df: df[:1000000])\n",
"bd.data[\"BAC\"].info()\n"
"# basic_data.wrapper.index.normalize().nunique()\n",
"# bd = basic_data.transform(lambda df: df[:1000000])\n",
"basic_data.data[\"BAC\"].info()\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"10"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"basic_data.wrapper.index.normalize().nunique()"
]
},
{
@ -569,19 +621,30 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
"from lib.db import Connection\n",
"SYMBOL = \"BAC\"\n",
"SCHEMA = \"ohlcv_1s\" #time based 1s other options ohlcv_vol_200 (volume based ohlcv with resolution of 200), ohlcv_renko_20 (renko with 20 bricks size) ...\n",
"DB = \"market_data\"\n",
"\n",
"try:\n",
" con = Connection(db_name=DB, default_schema=SCHEMA, create_db=True)\n",
" con.force_rollback_all() # Force cleanup before attempting to save\n",
" con.save(basic_data,1000)\n",
" con = Connection(db_name=DB, default_schema=SCHEMA, create_db=False)\n",
" #con.force_rollback_all() # Force cleanup before attempting to save\n",
" con.save(basic_data, chunksize=500)\n",
"except Exception as e:\n",
" print(f\"Error occurred: {str(e)}\")\n",
"\n",