flush db v1

.streamlit/secrets.toml

@@ -0,0 +1,4 @@
+# .streamlit/secrets.toml
+
+[connections.data_db]
+url = "sqlite:///data.db"

README.md

@@ -1,5 +1,13 @@
+万事胜意:
+你可以理解为,现在我上面写的所有train_data都是数据库初始化的值,都是不规则的所以要初始化时就填入,后续在streamlit 新添加的都会默认的把path填好 比如我填2023 KHQP 001 path就能有 
+
+万事胜意:
+lbs_path 不生成
+精度在初始化数据库都给S3
+
 ## UI
 
+
 ![](bak/ui_.png)
 
 ## reference

bak/evolve_config2.py

@@ -1,21 +1,17 @@
 update_cache = False
 train_data = [
     # khdp_7flys
-    dict(
+    {
-        img_prefix='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images',
+        'img_prefix': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images',
-        ann_file='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
+        'ann_file': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
-        select_ids=None,
+        'select_ids': None, 'filter_empty_gt': False, 'update_cache': False
-        filter_empty_gt=False,
+    },
-        update_cache=False,
-    ),
     # khdp_v2_3flys
-    dict(
+    {
-        img_prefix='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images',
+        'img_prefix': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images',
-        ann_file='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
+        'ann_file': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2',
-        select_ids=None,
+        'select_ids': None, 'filter_empty_gt': False, 'update_cache': False
-        filter_empty_gt=False,
+    },
-        update_cache=update_cache,
-    ),
     # khdp_v3
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_044_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
@@ -26,108 +22,130 @@ train_data = [
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     # khdp_v4
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     # khdp_202306_zhubin
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     {
         'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016',
         'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/images',
-        'filter_empty_gt': False, 'update_cache': update_cache},
+        'filter_empty_gt': False, 'update_cache': update_cache
+    },
 
     # KHXP_DZ
     {
@@ -472,3 +490,4 @@ ckpt_path = '/data/output/yolox_swinT_small_ds8x_s2048_s3_experiment42/backup_ck
 
 n_trail = 10
 n_epoch = 1
+

bak/init_data.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+from pydantic import BaseModel
+
+
+class BatchDataBase(BaseModel):
+    year: int | None = None
+    census_batch: str | None = None  # 普查批次
+    id_code: str | None = None  # 编号
+    precision: str | None = None  # 精度
+    is_train: bool | None = None
+    is_validation: bool | None = None
+
+    ann_file: str | None = None
+    img_prefix: str | None = None
+    filter_empty_gt: bool | None = None
+    update_cache: bool | None = None
+
+
+class BatchDataRead(BatchDataBase):
+    id: int
+
+    class Config:
+        orm_mode = True
+
+

flush.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+from loguru import logger
+from sqlalchemy import Column, Integer, String, create_engine
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+from bak.evolve_config2 import train_data
+from bak.init_data import BatchDataBase
+
+Base = declarative_base()
+# 创建 SQLite 数据库引擎
+engine = create_engine('sqlite:///data.db', echo=False)
+
+
+class BatchData(Base):
+    __tablename__ = 'batch_data'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    year = Column(Integer)
+    census_batch = Column(String)  # 普查批次
+    id_code = Column(String)
+    precision = Column(String)  # 精度
+    is_train = Column(Integer)
+    is_validation = Column(Integer)
+    ann_file = Column(String)
+    img_prefix = Column(String)
+    filter_empty_gt = Column(Integer)
+    update_cache = Column(Integer)
+
+
+Base.metadata.create_all(engine)
+
+Session = sessionmaker(bind=engine, )
+session = Session()
+logger.debug(f"{len(train_data)=}")
+
+census_batches = []
+for td in train_data:
+    db_obj = BatchData(**BatchDataBase(**td).dict())
+    batch_data_db = session.query(BatchData).filter_by(ann_file=db_obj.ann_file).first()
+    if '2023' in db_obj.ann_file:
+        batch_data_db.year = 2023
+    census_batch_idx = db_obj.ann_file.index('kh')
+    if census_batch_idx:
+        census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4]
+        census_batches.append(census_batch)
+        batch_data_db.census_batch = census_batch
+    # if not batch_data_db:
+    #     logger.debug(f"insert {db_obj.ann_file=}")
+    #     session.add(db_obj)
+
+session.commit()
+
+logger.info(f"{set(census_batches)=}")

main.py

@@ -1,16 +1,22 @@
-# 这是一个示例 Python 脚本。
+# streamlit_app.py
 
-# 按 Ctrl+F5 执行或将其替换为您的代码。
+import streamlit as st
-# 按 双击 Shift 在所有地方搜索类、文件、工具窗口、操作和设置。
 
+# Create the SQL connection to pets_db as specified in your secrets file.
+conn = st.connection('data_db', type='sql')
 
-def print_hi(name):
+# Insert some data with conn.session.
-    # 在下面的代码行中使用断点来调试脚本。
+with conn.session as s:
-    print(f'Hi, {name}')  # 按 F9 切换断点。
+    s.execute('CREATE TABLE IF NOT EXISTS pet_owners (person TEXT, pet TEXT);')
+    s.execute('DELETE FROM pet_owners;')
+    pet_owners = {'jerry': 'fish', 'barbara': 'cat', 'alex': 'puppy'}
+    for k in pet_owners:
+        s.execute(
+            'INSERT INTO pet_owners (person, pet) VALUES (:owner, :pet);',
+            params=dict(owner=k, pet=pet_owners[k])
+        )
+    s.commit()
 
-
+# Query and display the data you inserted
-# 按装订区域中的绿色按钮以运行脚本。
+pet_owners = conn.query('select * from pet_owners')
-if __name__ == '__main__':
+st.dataframe(pet_owners)
-    print_hi('PyCharm')
-
-# 访问 https://www.jetbrains.com/help/pycharm/ 获取 PyCharm 帮助

requirements.txt

@@ -0,0 +1,4 @@
+sqlalchemy~=1.4
+streamlit~=1.29
+pydantic~=1.0
+loguru~=0.7