diff --git a/.streamlit/secrets.toml b/.streamlit/secrets.toml new file mode 100644 index 0000000..b4b75cf --- /dev/null +++ b/.streamlit/secrets.toml @@ -0,0 +1,4 @@ +# .streamlit/secrets.toml + +[connections.data_db] +url = "sqlite:///data.db" \ No newline at end of file diff --git a/README.md b/README.md index 649993c..4ddfe8f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ +万事胜意: +你可以理解为,现在我上面写的所有train_data都是数据库初始化的值,都是不规则的所以要初始化时就填入,后续在streamlit 新添加的都会默认的把path填好 比如我填2023 KHQP 001 path就能有 + +万事胜意: +lbs_path 不生成 +精度在初始化数据库都给S3 + ## UI + ![](bak/ui_.png) ## reference diff --git a/bak/evolve_config2.py b/bak/evolve_config2.py index 79d3ddf..075b2e7 100644 --- a/bak/evolve_config2.py +++ b/bak/evolve_config2.py @@ -1,21 +1,17 @@ update_cache = False train_data = [ # khdp_7flys - dict( - img_prefix='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images', - ann_file='/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', - select_ids=None, - filter_empty_gt=False, - update_cache=False, - ), + { + 'img_prefix': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images', + 'ann_file': '/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', + 'select_ids': None, 'filter_empty_gt': False, 'update_cache': False + }, # khdp_v2_3flys - dict( - img_prefix='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images', - ann_file='/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', - select_ids=None, - filter_empty_gt=False, - update_cache=update_cache, - ), + { + 'img_prefix': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images', + 'ann_file': '/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2', + 'select_ids': None, 'filter_empty_gt': False, 'update_cache': False + }, # khdp_v3 { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_044_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', @@ -26,108 +22,130 @@ train_data = [ { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, # khdp_v4 { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, # khdp_202306_zhubin { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, { 'ann_file': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016', 'img_prefix': '/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/images', - 'filter_empty_gt': False, 'update_cache': update_cache}, + 'filter_empty_gt': False, 'update_cache': update_cache + }, # KHXP_DZ { @@ -472,3 +490,4 @@ ckpt_path = '/data/output/yolox_swinT_small_ds8x_s2048_s3_experiment42/backup_ck n_trail = 10 n_epoch = 1 + diff --git a/bak/init_data.py b/bak/init_data.py new file mode 100644 index 0000000..477e772 --- /dev/null +++ b/bak/init_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +from pydantic import BaseModel + + +class BatchDataBase(BaseModel): + year: int | None = None + census_batch: str | None = None # 普查批次 + id_code: str | None = None # 编号 + precision: str | None = None # 精度 + is_train: bool | None = None + is_validation: bool | None = None + + ann_file: str | None = None + img_prefix: str | None = None + filter_empty_gt: bool | None = None + update_cache: bool | None = None + + +class BatchDataRead(BatchDataBase): + id: int + + class Config: + orm_mode = True + + diff --git a/data.db b/data.db new file mode 100644 index 0000000..cd0c3b4 Binary files /dev/null and b/data.db differ diff --git a/flush.py b/flush.py new file mode 100644 index 0000000..c82c84f --- /dev/null +++ b/flush.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +from loguru import logger +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import declarative_base, sessionmaker + +from bak.evolve_config2 import train_data +from bak.init_data import BatchDataBase + +Base = declarative_base() +# 创建 SQLite 数据库引擎 +engine = create_engine('sqlite:///data.db', echo=False) + + +class BatchData(Base): + __tablename__ = 'batch_data' + id = Column(Integer, primary_key=True, autoincrement=True) + year = Column(Integer) + census_batch = Column(String) # 普查批次 + id_code = Column(String) + precision = Column(String) # 精度 + is_train = Column(Integer) + is_validation = Column(Integer) + ann_file = Column(String) + img_prefix = Column(String) + filter_empty_gt = Column(Integer) + update_cache = Column(Integer) + + +Base.metadata.create_all(engine) + +Session = sessionmaker(bind=engine, ) +session = Session() +logger.debug(f"{len(train_data)=}") + +census_batches = [] +for td in train_data: + db_obj = BatchData(**BatchDataBase(**td).dict()) + batch_data_db = session.query(BatchData).filter_by(ann_file=db_obj.ann_file).first() + if '2023' in db_obj.ann_file: + batch_data_db.year = 2023 + census_batch_idx = db_obj.ann_file.index('kh') + if census_batch_idx: + census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4] + census_batches.append(census_batch) + batch_data_db.census_batch = census_batch + # if not batch_data_db: + # logger.debug(f"insert {db_obj.ann_file=}") + # session.add(db_obj) + +session.commit() + +logger.info(f"{set(census_batches)=}") diff --git a/main.py b/main.py index e1d7a9b..dc87ef7 100644 --- a/main.py +++ b/main.py @@ -1,16 +1,22 @@ -# 这是一个示例 Python 脚本。 +# streamlit_app.py -# 按 Ctrl+F5 执行或将其替换为您的代码。 -# 按 双击 Shift 在所有地方搜索类、文件、工具窗口、操作和设置。 +import streamlit as st +# Create the SQL connection to pets_db as specified in your secrets file. +conn = st.connection('data_db', type='sql') -def print_hi(name): - # 在下面的代码行中使用断点来调试脚本。 - print(f'Hi, {name}') # 按 F9 切换断点。 +# Insert some data with conn.session. +with conn.session as s: + s.execute('CREATE TABLE IF NOT EXISTS pet_owners (person TEXT, pet TEXT);') + s.execute('DELETE FROM pet_owners;') + pet_owners = {'jerry': 'fish', 'barbara': 'cat', 'alex': 'puppy'} + for k in pet_owners: + s.execute( + 'INSERT INTO pet_owners (person, pet) VALUES (:owner, :pet);', + params=dict(owner=k, pet=pet_owners[k]) + ) + s.commit() - -# 按装订区域中的绿色按钮以运行脚本。 -if __name__ == '__main__': - print_hi('PyCharm') - -# 访问 https://www.jetbrains.com/help/pycharm/ 获取 PyCharm 帮助 +# Query and display the data you inserted +pet_owners = conn.query('select * from pet_owners') +st.dataframe(pet_owners) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0175e65 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +sqlalchemy~=1.4 +streamlit~=1.29 +pydantic~=1.0 +loguru~=0.7 \ No newline at end of file