leo/khqp_trainer
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

flush db v1

Browse Source
This commit is contained in:
leo
2024-03-06 20:30:27 +08:00
parent 08ef3f36a7
commit 31a2a947b4
8 changed files with 168 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.streamlit/secrets.toml Normal file
View File

@@ -0,0 +1,4 @@
# .streamlit/secrets.toml
[connections.data_db]
url = "sqlite:///data.db"

8
README.md
View File

@@ -1,5 +1,13 @@
万事胜意:
你可以理解为,现在我上面写的所有train_data都是数据库初始化的值,都是不规则的所以要初始化时就填入,后续在streamlit 新添加的都会默认的把path填好 比如我填2023 KHQP 001 path就能有
万事胜意:
lbs_path 不生成
精度在初始化数据库都给S3
## UI
![](bak/ui_.png)
## reference

91
bak/evolve_config2.py
View File

@@ -1,21 +1,17 @@
update_cache = False
train_data = [
# khdp_7flys
dict(
img_prefix='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images',
ann_file='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
select_ids=None,
filter_empty_gt=False,
update_cache=False,
),
{
'img_prefix': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images',
'ann_file': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
'select_ids': None, 'filter_empty_gt': False, 'update_cache': False
},
# khdp_v2_3flys
dict(
img_prefix='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images',
ann_file='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
select_ids=None,
filter_empty_gt=False,
update_cache=update_cache,
),
{
'img_prefix': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images',
'ann_file': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
'select_ids': None, 'filter_empty_gt': False, 'update_cache': False
},
# khdp_v3
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_044_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
@@ -26,108 +22,130 @@ train_data = [
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
# khdp_v4
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
# khdp_202306_zhubin
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
{
'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
'filter_empty_gt': False, 'update_cache': update_cache},
'filter_empty_gt': False, 'update_cache': update_cache
},
# KHXP_DZ
{
@@ -472,3 +490,4 @@ ckpt_path = '/data/output/yolox_swinT_small_ds8x_s2048_s3_experiment42/backup_ck
n_trail = 10
n_epoch = 1

26
bak/init_data.py Normal file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
from pydantic import BaseModel
class BatchDataBase(BaseModel):
year: int | None = None
census_batch: str | None = None # 普查批次
id_code: str | None = None # 编号
precision: str | None = None # 精度
is_train: bool | None = None
is_validation: bool | None = None
ann_file: str | None = None
img_prefix: str | None = None
filter_empty_gt: bool | None = None
update_cache: bool | None = None
class BatchDataRead(BatchDataBase):
id: int
class Config:
orm_mode = True

BIN
data.db Normal file
View File

Binary file not shown.

53
flush.py Normal file
View File

@@ -0,0 +1,53 @@
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
from loguru import logger
from sqlalchemy import Column, Integer, String, create_engine
from sqlalchemy.orm import declarative_base, sessionmaker
from bak.evolve_config2 import train_data
from bak.init_data import BatchDataBase
Base = declarative_base()
# 创建 SQLite 数据库引擎
engine = create_engine('sqlite:///data.db', echo=False)
class BatchData(Base):
__tablename__ = 'batch_data'
id = Column(Integer, primary_key=True, autoincrement=True)
year = Column(Integer)
census_batch = Column(String) # 普查批次
id_code = Column(String)
precision = Column(String) # 精度
is_train = Column(Integer)
is_validation = Column(Integer)
ann_file = Column(String)
img_prefix = Column(String)
filter_empty_gt = Column(Integer)
update_cache = Column(Integer)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine, )
session = Session()
logger.debug(f"{len(train_data)=}")
census_batches = []
for td in train_data:
db_obj = BatchData(**BatchDataBase(**td).dict())
batch_data_db = session.query(BatchData).filter_by(ann_file=db_obj.ann_file).first()
if '2023' in db_obj.ann_file:
batch_data_db.year = 2023
census_batch_idx = db_obj.ann_file.index('kh')
if census_batch_idx:
census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4]
census_batches.append(census_batch)
batch_data_db.census_batch = census_batch
# if not batch_data_db:
# logger.debug(f"insert {db_obj.ann_file=}")
# session.add(db_obj)
session.commit()
logger.info(f"{set(census_batches)=}")

30
main.py
View File

@@ -1,16 +1,22 @@
# 这是一个示例 Python 脚本。
# streamlit_app.py
# 按 Ctrl+F5 执行或将其替换为您的代码。
# 按 双击 Shift 在所有地方搜索类、文件、工具窗口、操作和设置。
import streamlit as st
# Create the SQL connection to pets_db as specified in your secrets file.
conn = st.connection('data_db', type='sql')
def print_hi(name):
# 在下面的代码行中使用断点来调试脚本。
print(f'Hi, {name}') # 按 F9 切换断点。
# Insert some data with conn.session.
with conn.session as s:
s.execute('CREATE TABLE IF NOT EXISTS pet_owners (person TEXT, pet TEXT);')
s.execute('DELETE FROM pet_owners;')
pet_owners = {'jerry': 'fish', 'barbara': 'cat', 'alex': 'puppy'}
for k in pet_owners:
s.execute(
'INSERT INTO pet_owners (person, pet) VALUES (:owner, :pet);',
params=dict(owner=k, pet=pet_owners[k])
)
s.commit()
# 按装订区域中的绿色按钮以运行脚本。
if __name__ == '__main__':
print_hi('PyCharm')
# 访问 https://www.jetbrains.com/help/pycharm/ 获取 PyCharm 帮助
# Query and display the data you inserted
pet_owners = conn.query('select * from pet_owners')
st.dataframe(pet_owners)

4
requirements.txt Normal file
View File

@@ -0,0 +1,4 @@
sqlalchemy~=1.4
streamlit~=1.29
pydantic~=1.0
loguru~=0.7