diff --git a/data.db b/data.db index e4b6472..97a4608 100644 Binary files a/data.db and b/data.db differ diff --git a/flush.py b/flush.py index ef63f8f..ce32e76 100644 --- a/flush.py +++ b/flush.py @@ -2,36 +2,39 @@ # -*- coding:utf-8 -*- from loguru import logger -from bak.evolve_config2 import train_data -from bak.init_data import BatchDataBase +from bak.evolve_config2 import train_data, val_data +from bak.init_data import BatchDataBase, BatchDataRead from db_utils import BatchData, session -logger.debug(f"{len(train_data)=}") - -census_batches = [] -for td in train_data: +logger.debug(f"{len(val_data)=}") +for td in val_data: db_obj = BatchData(**BatchDataBase(**td).dict()) - batch_data_db = session.query(BatchData).filter_by(ann_file=db_obj.ann_file).first() - if '2023' in db_obj.ann_file: - batch_data_db.year = 2023 - census_batch_idx = db_obj.ann_file.index('kh') - if census_batch_idx: - census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4].upper() - census_batches.append(census_batch) - batch_data_db.census_batch = census_batch + batch_data_db = session.query(BatchData).where(BatchData.img_prefix == td.get("img_prefix")).first() + if batch_data_db: + logger.debug(f"{BatchDataRead.from_orm(batch_data_db)=}") + continue + session.add(db_obj) - batch_data_db.precision = 'S3' - batch_data_db.is_train = True - batch_data_db.is_validation = False - - id_code_idx = census_batch_idx + 5 - if batch_data_db.id_code is None: - if batch_data_db.id > 29: - code_start = db_obj.ann_file[id_code_idx + 21:] - code_end = code_start.index('_') - id_code = db_obj.ann_file[id_code_idx + 21:][:code_end] - logger.debug(f"{id_code}, {db_obj.ann_file[id_code_idx + 21:]}, {db_obj.ann_file[id_code_idx:]}, {batch_data_db.id=}") - batch_data_db.id_code = id_code +# if '2023' in db_obj.ann_file: +# batch_data_db.year = 2023 +# census_batch_idx = db_obj.ann_file.index('kh') +# if census_batch_idx: +# census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4].upper() +# census_batches.append(census_batch) +# batch_data_db.census_batch = census_batch +# +# batch_data_db.precision = 'S3' +# batch_data_db.is_train = True +# batch_data_db.is_validation = False +# +# id_code_idx = census_batch_idx + 5 +# if batch_data_db.id_code is None: +# if batch_data_db.id > 29: +# code_start = db_obj.ann_file[id_code_idx + 21:] +# code_end = code_start.index('_') +# id_code = db_obj.ann_file[id_code_idx + 21:][:code_end] +# logger.debug(f"{id_code}, {db_obj.ann_file[id_code_idx + 21:]}, {db_obj.ann_file[id_code_idx:]}, {batch_data_db.id=}") +# batch_data_db.id_code = id_code # if batch_data_db.id > 25 and batch_data_db.id <= 29: # code_start = db_obj.ann_file[id_code_idx + 18:] @@ -55,4 +58,3 @@ for td in train_data: session.commit() -logger.info(f"{set(census_batches)=}")