diff --git a/bak/batch_data.csv b/bak/batch_data.csv new file mode 100644 index 0000000..c90e2ba --- /dev/null +++ b/bak/batch_data.csv @@ -0,0 +1,71 @@ +id,year,census_batch,id_code,precision,is_train,is_validation,ann_file,img_prefix,filter_empty_gt,update_cache +1,2023,KHDP,,S3,1,0,/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2,/data/data/split_ss_yimu_m6_p2/train_khdp_7flys_gsd5_orgsize_noDecay_seg-v2/images,0,0 +2,2023,KHDP,,S3,1,0,/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230531_p1p2,/data/data/split_ss_yimu_m6_p1/train_khdp_v2_3flys_gsd5_orgsize_noDecay_seg-v2/images,0,0 +3,2023,KHDP,044,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_044_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_044_gsd5_orgsize_noDecay_seg-v2/images,0,0 +4,2023,KHDP,064,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_064_gsd5_orgsize_noDecay_seg-v2/images,0,0 +5,2023,KHDP,065,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_065_gsd5_orgsize_noDecay_seg-v2/images,0,0 +6,2023,KHDP,073,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_073_gsd5_orgsize_noDecay_seg-v2/images,0,0 +7,2023,KHDP,072,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_072_gsd5_orgsize_noDecay_seg-v2/images,0,0 +8,2023,KHDP,013,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_013_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +9,2023,KHDP,015,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_015_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +10,2023,KHDP,016,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_016_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +11,2023,KHDP,028,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_20230601,/data/data/split_ss_yimu_m6_p4/test_khdp_028_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +12,2023,KHDP,067,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_067_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +13,2023,KHDP,068,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_068_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +14,2023,KHDP,069,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_069_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +15,2023,KHDP,070,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_070_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +16,2023,KHDP,071,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_071_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +17,2023,KHDP,075,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_075_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +18,2023,KHDP,078,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_078_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +19,2023,KHDP,080,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_080_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +20,2023,KHDP,081,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_081_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +21,2023,KHDP,082,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_082_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +22,2023,KHDP,084,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_084_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +23,2023,KHDP,087,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_087_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +24,2023,KHDP,089,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_089_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +25,2023,KHDP,090,S3,1,0,/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/annfiles2_labelme_202306016,/data/data/split_ss_yimu_m6_p4/test_khdp_090_mod_loc_gsd5_orgsize_noDecay_seg-v2/images,0,0 +26,2023,KHXP,001a002,S3,1,0,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_001a002_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_001a002_gsd5_s2048/images,0,0 +27,2023,KHXP,003a004a006,S3,1,0,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_003a004a006_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_003a004a006_gsd5_s2048/images,0,0 +28,2023,KHXP,005,S3,1,0,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_005_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_005_gsd5_s2048/images,0,0 +29,2023,KHXP,007a008,S3,1,0,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_007a008_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khxpdz/patch_2023_khdz_007a008_gsd5_s2048/images,0,0 +30,2023,KHQP,001a006a010,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_001a006a010_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_001a006a010_gsd5_s2048/images,0,0 +31,2023,KHQP,002a004,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_002a004_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_002a004_gsd5_s2048/images,0,0 +32,2023,KHQP,003,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_003_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_003_gsd5_s2048/images,0,0 +33,2023,KHQP,005,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_005_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_005_gsd5_s2048/images,0,0 +34,2023,KHQP,007a008a009,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_007a008a009_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_007a008a009_gsd5_s2048/images,0,0 +35,2023,KHQP,011,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_011_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_011_gsd5_s2048/images,0,0 +36,2023,KHQP,012a013,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_012a013_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_012a013_gsd5_s2048/images,0,0 +37,2023,KHQP,014a038,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_014a038_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_014a038_gsd5_s2048/images,0,0 +38,2023,KHQP,015a016,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_015a016_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_015a016_gsd5_s2048/images,0,0 +39,2023,KHQP,018a069,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_018a069_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_018a069_gsd5_s2048/images,0,0 +40,2023,KHQP,019a075,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_019a075_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_019a075_gsd5_s2048/images,0,0 +41,2023,KHQP,020a022,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_020a022_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_020a022_gsd5_s2048/images,0,0 +42,2023,KHQP,023a025,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_023a025_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_023a025_gsd5_s2048/images,0,0 +43,2023,KHQP,026a029a079,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_026a029a079_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_026a029a079_gsd5_s2048/images,0,0 +44,2023,KHQP,028a027a030,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_028a027a030_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_028a027a030_gsd5_s2048/images,0,0 +45,2023,KHQP,032a033,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_032a033_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_032a033_gsd5_s2048/images,0,0 +46,2023,KHQP,039a040,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_039a040_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_039a040_gsd5_s2048/images,0,0 +47,2023,KHQP,042a045,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_042a045_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_042a045_gsd5_s2048/images,0,0 +48,2023,KHQP,046a049,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_046a049_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_046a049_gsd5_s2048/images,0,0 +49,2023,KHQP,050a058,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_050a058_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_050a058_gsd5_s2048/images,0,0 +50,2023,KHQP,053a059a060,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_053a059a060_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_053a059a060_gsd5_s2048/images,0,0 +51,2023,KHQP,054a056,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_054a056_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_054a056_gsd5_s2048/images,0,0 +52,2023,KHQP,063a143,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_063a143_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_063a143_gsd5_s2048/images,0,0 +53,2023,KHQP,073a074a080,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_073a074a080_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_073a074a080_gsd5_s2048/images,0,0 +54,2023,KHQP,072a081a082,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_072a081a082_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_072a081a082_gsd5_s2048/images,0,0 +55,2023,KHQP,076,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_076_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_076_gsd5_s2048/images,0,0 +56,2023,KHQP,077a078,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_077a078_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_077a078_gsd5_s2048/images,0,0 +57,2023,KHQP,091a097a102a104,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_091a097a102a104_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_091a097a102a104_gsd5_s2048/images,0,0 +58,2023,KHQP,087a088a089a090,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_087a088a089a090_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_087a088a089a090_gsd5_s2048/images,0,0 +59,2023,KHQP,094,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_094_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_094_gsd5_s2048/images,0,0 +60,2023,KHQP,105a0111,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_105a0111_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_105a0111_gsd5_s2048/images,0,0 +61,2023,KHQP,112a113,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_112a113_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_112a113_gsd5_s2048/images,0,0 +62,2023,KHQP,114a116a118,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_114a116a118_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_114a116a118_gsd5_s2048/images,0,0 +63,2023,KHQP,115,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_115_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_115_gsd5_s2048/images,0,0 +64,2023,KHQP,117,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_117_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_117_gsd5_s2048/images,0,0 +65,2023,KHQP,119a120,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_119a120_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_119a120_gsd5_s2048/images,0,0 +66,2023,KHQP,034a131,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_034a131_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_034a131_gsd5_s2048/images,0,0 +67,2023,KHQP,132a133,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_132a133_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_132a133_gsd5_s2048/images,0,0 +68,2023,KHQP,057,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_057_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_057_gsd5_s2048/images,0,0 +69,2023,KHQP,066a142,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_066a142_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_066a142_gsd5_s2048/images,0,0 +70,2023,KHQP,106a107a138a139,S3,1,0,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_106a107a138a139_gsd5_s2048/annfiles,/data/data/split_ss_yimu_khqp_2023/patch_2023_khqp_106a107a138a139_gsd5_s2048/images,0,0 diff --git a/data.db b/data.db index cd0c3b4..a0de274 100644 Binary files a/data.db and b/data.db differ diff --git a/flush.py b/flush.py index c82c84f..75b8369 100644 --- a/flush.py +++ b/flush.py @@ -41,9 +41,39 @@ for td in train_data: batch_data_db.year = 2023 census_batch_idx = db_obj.ann_file.index('kh') if census_batch_idx: - census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4] + census_batch = db_obj.ann_file[census_batch_idx:census_batch_idx + 4].upper() census_batches.append(census_batch) batch_data_db.census_batch = census_batch + + batch_data_db.precision = 'S3' + batch_data_db.is_train = True + batch_data_db.is_validation = False + + id_code_idx = census_batch_idx + 5 + if batch_data_db.id_code is None: + if batch_data_db.id > 29: + code_start = db_obj.ann_file[id_code_idx + 21:] + code_end = code_start.index('_') + id_code = db_obj.ann_file[id_code_idx + 21:][:code_end] + logger.debug(f"{id_code}, {db_obj.ann_file[id_code_idx + 21:]}, {db_obj.ann_file[id_code_idx:]}, {batch_data_db.id=}") + batch_data_db.id_code = id_code + + # if batch_data_db.id > 25 and batch_data_db.id <= 29: + # code_start = db_obj.ann_file[id_code_idx + 18:] + # code_end = code_start.index('_') + # id_code = db_obj.ann_file[id_code_idx + 18:][:code_end] + # logger.debug(f"{db_obj.ann_file[id_code_idx + 18:]}, {db_obj.ann_file[id_code_idx:]}, {id_code}, {batch_data_db.id=}") + # batch_data_db.id_code = id_code + # if 2 < batch_data_db.id <= 25: + # id_code = db_obj.ann_file[id_code_idx:id_code_idx + 3] + # logger.debug(f"{id_code=}") + # batch_data_db.id_code = id_code + + # if 25 < batch_data_db.id <= 25: + # id_code = db_obj.ann_file[id_code_idx:id_code_idx + 3] + # logger.debug(f"{id_code=}") + # batch_data_db.id_code = id_code + # if not batch_data_db: # logger.debug(f"insert {db_obj.ann_file=}") # session.add(db_obj)