leo/khqp_trainer
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

更新数据

Browse Source
This commit is contained in:
leo
2024-03-07 19:47:24 +08:00
parent 471698f5a4
commit 9d0380c50b
2 changed files with 29 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
data.db
View File

Binary file not shown.

56
flush.py
View File

@@ -2,36 +2,39 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
from loguru import logger from loguru import logger
from bak.evolve_config2 import train_data from bak.evolve_config2 import train_data, val_data
from bak.init_data import BatchDataBase from bak.init_data import BatchDataBase, BatchDataRead
from db_utils import BatchData, session from db_utils import BatchData, session
logger.debug(f"{len(train_data)=}") logger.debug(f"{len(val_data)=}")
for td in val_data:
census_batches = []
for td in train_data:
db_obj = BatchData(**BatchDataBase(**td).dict()) db_obj = BatchData(**BatchDataBase(**td).dict())
batch_data_db = session.query(BatchData).filter_by(ann_file=db_obj.ann_file).first() batch_data_db = session.query(BatchData).where(BatchData.img_prefix == td.get("img_prefix")).first()
if '2023' in db_obj.ann_file: if batch_data_db:
batch_data_db.year = 2023 logger.debug(f"{BatchDataRead.from_orm(batch_data_db)=}")
census_batch_idx = db_obj.ann_file.index('kh') continue
if census_batch_idx: session.add(db_obj)
census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4].upper()
census_batches.append(census_batch)
batch_data_db.census_batch = census_batch
batch_data_db.precision = 'S3' # if '2023' in db_obj.ann_file:
batch_data_db.is_train = True # batch_data_db.year = 2023
batch_data_db.is_validation = False # census_batch_idx = db_obj.ann_file.index('kh')
# if census_batch_idx:
id_code_idx = census_batch_idx + 5 # census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4].upper()
if batch_data_db.id_code is None: # census_batches.append(census_batch)
if batch_data_db.id > 29: # batch_data_db.census_batch = census_batch
code_start = db_obj.ann_file[id_code_idx + 21:] #
code_end = code_start.index('_') # batch_data_db.precision = 'S3'
id_code = db_obj.ann_file[id_code_idx + 21:][:code_end] # batch_data_db.is_train = True
logger.debug(f"{id_code}, {db_obj.ann_file[id_code_idx + 21:]}, {db_obj.ann_file[id_code_idx:]}, {batch_data_db.id=}") # batch_data_db.is_validation = False
batch_data_db.id_code = id_code #
# id_code_idx = census_batch_idx + 5
# if batch_data_db.id_code is None:
# if batch_data_db.id > 29:
# code_start = db_obj.ann_file[id_code_idx + 21:]
# code_end = code_start.index('_')
# id_code = db_obj.ann_file[id_code_idx + 21:][:code_end]
# logger.debug(f"{id_code}, {db_obj.ann_file[id_code_idx + 21:]}, {db_obj.ann_file[id_code_idx:]}, {batch_data_db.id=}")
# batch_data_db.id_code = id_code
# if batch_data_db.id > 25 and batch_data_db.id <= 29: # if batch_data_db.id > 25 and batch_data_db.id <= 29:
# code_start = db_obj.ann_file[id_code_idx + 18:] # code_start = db_obj.ann_file[id_code_idx + 18:]
@@ -55,4 +58,3 @@ for td in train_data:
session.commit() session.commit()
logger.info(f"{set(census_batches)=}")