From 31a2a947b41cfa8f237b2b192904b5887bc33bb3 Mon Sep 17 00:00:00 2001 From: leo Date: Wed, 6 Mar 2024 20:30:27 +0800 Subject: [PATCH] flush db v1 --- .streamlit/secrets.toml | 4 ++ README.md | 8 ++++ bak/evolve_config2.py | 91 ++++++++++++++++++++++++---------------- bak/init_data.py | 26 ++++++++++++ data.db | Bin 0 -> 28672 bytes flush.py | 53 +++++++++++++++++++++++ main.py | 30 +++++++------ requirements.txt | 4 ++ 8 files changed, 168 insertions(+), 48 deletions(-) create mode 100644 .streamlit/secrets.toml create mode 100644 bak/init_data.py create mode 100644 data.db create mode 100644 flush.py create mode 100644 requirements.txt diff --git a/.streamlit/secrets.toml b/.streamlit/secrets.toml new file mode 100644 index 0000000..b4b75cf --- /dev/null +++ b/.streamlit/secrets.toml @@ -0,0 +1,4 @@ +# .streamlit/secrets.toml + +[connections.data_db] +url = "sqlite:///data.db" \ No newline at end of file diff --git a/README.md b/README.md index 649993c..4ddfe8f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ +万事胜意: +你可以理解为,现在我上面写的所有train_data都是数据库初始化的值,都是不规则的所以要初始化时就填入,后续在streamlit 新添加的都会默认的把path填好 比如我填2023 KHQP 001 path就能有 + +万事胜意: +lbs_path 不生成 +精度在初始化数据库都给S3 + ## UI + ![](bak/ui_.png) ## reference diff --git a/bak/evolve_config2.py b/bak/evolve_config2.py index 79d3ddf..075b2e7 100644 --- a/bak/evolve_config2.py +++ b/bak/evolve_config2.py @@ -1,21 +1,17 @@ update_cache = False train_data = [ # khdp_7flys - dict( - img_prefix='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images', - ann_file='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', - select_ids=None, - filter_empty_gt=False, - update_cache=False, - ), + { + 'img_prefix': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images', + 'ann_file': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', + 'select_ids': None, 'filter_empty_gt': False, 'update_cache': False + }, # khdp_v2_3flys - dict( - img_prefix='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images', - ann_file='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', - select_ids=None, - filter_empty_gt=False, - update_cache=update_cache, - ), + { + 'img_prefix': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images', + 'ann_file': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', + 'select_ids': None, 'filter_empty_gt': False, 'update_cache': False + }, # khdp_v3 { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_044_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', @@ -26,108 +22,130 @@ train_data = [ { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, # khdp_v4 { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, # khdp_202306_zhubin { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, # KHXP_DZ { @@ -472,3 +490,4 @@ ckpt_path = '/data/output/yolox_swinT_small_ds8x_s2048_s3_experiment42/backup_ck n_trail = 10 n_epoch = 1 + diff --git a/bak/init_data.py b/bak/init_data.py new file mode 100644 index 0000000..477e772 --- /dev/null +++ b/bak/init_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +from pydantic import BaseModel + + +class BatchDataBase(BaseModel): + year: int | None = None + census_batch: str | None = None # 普查批次 + id_code: str | None = None # 编号 + precision: str | None = None # 精度 + is_train: bool | None = None + is_validation: bool | None = None + + ann_file: str | None = None + img_prefix: str | None = None + filter_empty_gt: bool | None = None + update_cache: bool | None = None + + +class BatchDataRead(BatchDataBase): + id: int + + class Config: + orm_mode = True + + diff --git a/data.db b/data.db new file mode 100644 index 0000000000000000000000000000000000000000..cd0c3b4be5eeb5f47dee7f5e95920554699b1379 GIT binary patch literal 28672 zcmeHP&2JmW6*Z-cjZqB?eWnRxTy>1My2N4J&5JVt|KoEf-0zm|V2m}!bA`nC% zh`?tk0?(Zd$B!I47JB|dN58Rc+|WC#xVE%#Whr-M;r#Ma?vv}e*~w(1 zp1W}A%F>fdYq?8LU&&p%y8PVPkXVDubGganzM=0CPuB06vD3cU*1BMF&n&Dheq&+H zy56X3wcWbWd#SZ&)Ee!^?oRK;MqBIb>5ZLUn{e%xzTE)w*3FofWmMnU(bgN=eO4RI z4Gk=>H*WWug}XY&o@O*#oqcVi(`)2r3k(}tO|NYlz1LUPE3VLqQ9n6( z(yGm;LvKsjxy)Mng{j}BK1}5&jpX+dKac-09*UiaJ~h#f+ztOG^mZsEp>8gQej$bL zv%Z;z`B)+m|LfLfy)|z&qy5otZ8thvyRGdvnm4tksB`mlD50I(Qe!^w4L3@jheo%wT+F_x0HE^gB6ChqHXIpjP0hODY88)H8!v-u*-GJMR` z>iF7>(HCKs=43X;5kzJ4MVL)-GAoa<&DeZlX3wN3vznjzOks z7lcTRlStw4i41l@h(!5_sHH}bAQAlnt~5fd||ms$BxX^rU0 z=G5KP&%SGw^U>^ovL9sM&AvDJ!{o0Me@omSUySWWU!3?|0js0b= zelCNZ+KR)##{HCN4XdCv*soDJF;@QG{e1-egwYr29-uP#9M#k3QfFWgQmpdesQT?K)Ceqt=>y*UYYMJR!G;ExhxDKCN&qbouQR0E%s*k%kN zl-OJmN}w0`q{LRf3!%j33R5}d`JN=g31sVgaT*;J}C??-GxvX?1E4L zb-*8mAw|0g3PW5F3ZM)4q;U9!v^v-Yp#X}2KMF$%brG#F#093X3R(cB(A5G4cIVOS zat|Vhs{&jE6##SUssIByjaZcP5E|+VK*x&k5kChX6UX5rJe`iNNcT!bDe-*#)rnh? zm%?v^-T{8a*#Bb7(O0rpGcTq8kv>0lH}$7fIreJwqv%R>#k}?a)j@n60jRPq?LC4q z+7SRoUpQZ{eSwczWsF;sjJ^o7Bb?0Gmc&BZtitAtFuR|VS&?(sWQ1a=!sd%GyN{EZ zGDc=>z6i6=b24KSJj7%pn=hML@>%J+6o#P{jz^s|AF(uI&-r0%^INU%siyf>_xzi6 zzzHO9xF{>KE-QG`tF6ez%DkET5)Ijd<`G+bzU_VPYHRa$tA3~7a{Z1b%Xs6)wn_pt z03(YT+^X_}6yhoKgnx(yU6zaRR_!eSUZ}3-Fyz@UQpm^6qy8ZyfZNwz{v+!!MfwcL z;K9K=w99eVbIPHwsy)|$T7)BYJgybl{B|K`V4^yDHas2{y zZw?;7?iMldnPU_@3bWm1g{_hK?H&b1Q7@#5JVt|KoEf-0zm{w1kC(-y1wVEqzXkH z!I|amld$}il+zk1{T2iPMyRSIMN}2#V$B8&k^V7}cdBN|rzVA<Z; z$`@HwP#|afWE>8u&BdAxI6bi?O-GVaK-^PL`@wsOdSWT*??gx{;q)0KrO)f z)AzJt=8K}D#s}o!`5&h>(#!=>fJ@LYC1H8#w1xqz3I)u$hgX7WwFJYJDysUF-7S(x zBMO{zJy3!U&0m(%}*ntv3!%F!sP2*Kawq_ zuK#%po&4=Kxkr zRTP0LoF1>j!2}t0y~G`)_qK`&0eg7Jx^g1V!FyjpTNj0_M!^vM{N$-x3C@ ZsuU>S2rZXE8>v8gRD_+XaJOg!|37APy($0z literal 0 HcmV?d00001 diff --git a/flush.py b/flush.py new file mode 100644 index 0000000..c82c84f --- /dev/null +++ b/flush.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +from loguru import logger +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import declarative_base, sessionmaker + +from bak.evolve_config2 import train_data +from bak.init_data import BatchDataBase + +Base = declarative_base() +# 创建 SQLite 数据库引擎 +engine = create_engine('sqlite:///data.db', echo=False) + + +class BatchData(Base): + __tablename__ = 'batch_data' + id = Column(Integer, primary_key=True, autoincrement=True) + year = Column(Integer) + census_batch = Column(String) # 普查批次 + id_code = Column(String) + precision = Column(String) # 精度 + is_train = Column(Integer) + is_validation = Column(Integer) + ann_file = Column(String) + img_prefix = Column(String) + filter_empty_gt = Column(Integer) + update_cache = Column(Integer) + + +Base.metadata.create_all(engine) + +Session = sessionmaker(bind=engine, ) +session = Session() +logger.debug(f"{len(train_data)=}") + +census_batches = [] +for td in train_data: + db_obj = BatchData(**BatchDataBase(**td).dict()) + batch_data_db = session.query(BatchData).filter_by(ann_file=db_obj.ann_file).first() + if '2023' in db_obj.ann_file: + batch_data_db.year = 2023 + census_batch_idx = db_obj.ann_file.index('kh') + if census_batch_idx: + census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4] + census_batches.append(census_batch) + batch_data_db.census_batch = census_batch + # if not batch_data_db: + # logger.debug(f"insert {db_obj.ann_file=}") + # session.add(db_obj) + +session.commit() + +logger.info(f"{set(census_batches)=}") diff --git a/main.py b/main.py index e1d7a9b..dc87ef7 100644 --- a/main.py +++ b/main.py @@ -1,16 +1,22 @@ -# 这是一个示例 Python 脚本。 +# streamlit_app.py -# 按 Ctrl+F5 执行或将其替换为您的代码。 -# 按 双击 Shift 在所有地方搜索类、文件、工具窗口、操作和设置。 +import streamlit as st +# Create the SQL connection to pets_db as specified in your secrets file. +conn = st.connection('data_db', type='sql') -def print_hi(name): - # 在下面的代码行中使用断点来调试脚本。 - print(f'Hi, {name}') # 按 F9 切换断点。 +# Insert some data with conn.session. +with conn.session as s: + s.execute('CREATE TABLE IF NOT EXISTS pet_owners (person TEXT, pet TEXT);') + s.execute('DELETE FROM pet_owners;') + pet_owners = {'jerry': 'fish', 'barbara': 'cat', 'alex': 'puppy'} + for k in pet_owners: + s.execute( + 'INSERT INTO pet_owners (person, pet) VALUES (:owner, :pet);', + params=dict(owner=k, pet=pet_owners[k]) + ) + s.commit() - -# 按装订区域中的绿色按钮以运行脚本。 -if __name__ == '__main__': - print_hi('PyCharm') - -# 访问 https://www.jetbrains.com/help/pycharm/ 获取 PyCharm 帮助 +# Query and display the data you inserted +pet_owners = conn.query('select * from pet_owners') +st.dataframe(pet_owners) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0175e65 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +sqlalchemy~=1.4 +streamlit~=1.29 +pydantic~=1.0 +loguru~=0.7 \ No newline at end of file