init

backend/inpaint/lama/fetch_data/celebahq_dataset_prepare.sh

@@ -0,0 +1,37 @@
+mkdir celeba-hq-dataset
+
+unzip data256x256.zip -d celeba-hq-dataset/
+
+# Reindex
+for i in `echo {00001..30000}`
+do
+    mv 'celeba-hq-dataset/data256x256/'$i'.jpg' 'celeba-hq-dataset/data256x256/'$[10#$i - 1]'.jpg'
+done
+
+
+# Split: split train -> train & val
+cat fetch_data/train_shuffled.flist | shuf > celeba-hq-dataset/temp_train_shuffled.flist
+cat celeba-hq-dataset/temp_train_shuffled.flist | head -n 2000 > celeba-hq-dataset/val_shuffled.flist
+cat celeba-hq-dataset/temp_train_shuffled.flist | tail -n +2001 > celeba-hq-dataset/train_shuffled.flist
+cat fetch_data/val_shuffled.flist > celeba-hq-dataset/visual_test_shuffled.flist
+
+mkdir celeba-hq-dataset/train_256/
+mkdir celeba-hq-dataset/val_source_256/
+mkdir celeba-hq-dataset/visual_test_source_256/
+
+cat celeba-hq-dataset/train_shuffled.flist | xargs -I {} mv celeba-hq-dataset/data256x256/{} celeba-hq-dataset/train_256/
+cat celeba-hq-dataset/val_shuffled.flist | xargs -I {} mv celeba-hq-dataset/data256x256/{} celeba-hq-dataset/val_source_256/
+cat celeba-hq-dataset/visual_test_shuffled.flist | xargs -I {} mv celeba-hq-dataset/data256x256/{} celeba-hq-dataset/visual_test_source_256/
+
+
+# create location config celeba.yaml
+PWD=$(pwd)
+DATASET=${PWD}/celeba-hq-dataset
+CELEBA=${PWD}/configs/training/location/celeba.yaml
+
+touch $CELEBA
+echo "# @package _group_" >> $CELEBA
+echo "data_root_dir: ${DATASET}/" >> $CELEBA
+echo "out_root_dir: ${PWD}/experiments/" >> $CELEBA
+echo "tb_dir: ${PWD}/tb_logs/" >> $CELEBA
+echo "pretrained_models: ${PWD}/" >> $CELEBA

backend/inpaint/lama/fetch_data/celebahq_gen_masks.sh

@@ -0,0 +1,29 @@
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thick_256.yaml \
+celeba-hq-dataset/val_source_256/ \
+celeba-hq-dataset/val_256/random_thick_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thin_256.yaml \
+celeba-hq-dataset/val_source_256/ \
+celeba-hq-dataset/val_256/random_thin_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_medium_256.yaml \
+celeba-hq-dataset/val_source_256/ \
+celeba-hq-dataset/val_256/random_medium_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thick_256.yaml \
+celeba-hq-dataset/visual_test_source_256/ \
+celeba-hq-dataset/visual_test_256/random_thick_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thin_256.yaml \
+celeba-hq-dataset/visual_test_source_256/ \
+celeba-hq-dataset/visual_test_256/random_thin_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_medium_256.yaml \
+celeba-hq-dataset/visual_test_source_256/ \
+celeba-hq-dataset/visual_test_256/random_medium_256/

backend/inpaint/lama/fetch_data/eval_sampler.py

@@ -0,0 +1,20 @@
+import os
+import random
+
+val_files_path           = os.path.abspath('') + '/places_standard_dataset/original/val/'
+list_of_random_val_files = os.path.abspath('') + '/places_standard_dataset/original/eval_random_files.txt'
+val_files      = [val_files_path + image for image in os.listdir(val_files_path)]
+
+print(f'Sampling 30000 images out of {len(val_files)} images in {val_files_path}' + \
+      f'and put their paths to {list_of_random_val_files}')
+
+print('In our paper we evaluate trained models on these 30k sampled (mask,image) pairs in our paper (check Sup. mat.)')
+
+random.shuffle(val_files)
+val_files_random = val_files[0:30000]
+
+with open(list_of_random_val_files, 'w') as fw:
+    for filename in val_files_random:
+        fw.write(filename+'\n')
+print('...done')      
+

backend/inpaint/lama/fetch_data/places_challenge_train_download.sh

@@ -0,0 +1,14 @@
+mkdir places_challenge_dataset
+
+
+declare -a TARPARTS
+for i in {a..z}
+do
+    TARPARTS[${#TARPARTS[@]}]="http://data.csail.mit.edu/places/places365/train_large_split/${i}.tar"
+done    
+ls
+printf "%s\n" "${TARPARTS[@]}" > places_challenge_dataset/places365_train.txt
+
+cd places_challenge_dataset/
+xargs -a places365_train.txt -n 1 -P 8 wget [...]
+ls *.tar | xargs -i tar xvf {}

backend/inpaint/lama/fetch_data/places_standard_evaluation_prepare_data.sh

@@ -0,0 +1,52 @@
+# 0. folder preparation
+mkdir -p places_standard_dataset/evaluation/hires/
+mkdir -p places_standard_dataset/evaluation/random_thick_512/
+mkdir -p places_standard_dataset/evaluation/random_thin_512/
+mkdir -p places_standard_dataset/evaluation/random_medium_512/
+mkdir -p places_standard_dataset/evaluation/random_thick_256/
+mkdir -p places_standard_dataset/evaluation/random_thin_256/
+mkdir -p places_standard_dataset/evaluation/random_medium_256/
+
+# 1. sample 30000 new images
+OUT=$(python3 fetch_data/eval_sampler.py)
+echo ${OUT}
+
+FILELIST=$(cat places_standard_dataset/original/eval_random_files.txt)
+for i in $FILELIST
+do
+    $(cp ${i} places_standard_dataset/evaluation/hires/)
+done
+
+
+# 2. generate all kinds of masks
+
+# all 512
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thick_512.yaml \
+places_standard_dataset/evaluation/hires \
+places_standard_dataset/evaluation/random_thick_512/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thin_512.yaml \
+places_standard_dataset/evaluation/hires \
+places_standard_dataset/evaluation/random_thin_512/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_medium_512.yaml \
+places_standard_dataset/evaluation/hires \
+places_standard_dataset/evaluation/random_medium_512/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thick_256.yaml \
+places_standard_dataset/evaluation/hires \
+places_standard_dataset/evaluation/random_thick_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thin_256.yaml \
+places_standard_dataset/evaluation/hires \
+places_standard_dataset/evaluation/random_thin_256/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_medium_256.yaml \
+places_standard_dataset/evaluation/hires \
+places_standard_dataset/evaluation/random_medium_256/

backend/inpaint/lama/fetch_data/places_standard_test_val_gen_masks.sh

@@ -0,0 +1,13 @@
+mkdir -p places_standard_dataset/val/
+mkdir -p places_standard_dataset/visual_test/
+
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thick_512.yaml \
+places_standard_dataset/val_hires/ \
+places_standard_dataset/val/
+
+python3 bin/gen_mask_dataset.py \
+$(pwd)/configs/data_gen/random_thick_512.yaml \
+places_standard_dataset/visual_test_hires/ \
+places_standard_dataset/visual_test/

backend/inpaint/lama/fetch_data/places_standard_test_val_prepare.sh

@@ -0,0 +1,5 @@
+mkdir -p places_standard_dataset/original/test/
+tar -xvf test_large.tar -C places_standard_dataset/original/test/
+
+mkdir -p places_standard_dataset/original/val/
+tar -xvf val_large.tar -C places_standard_dataset/original/val/

backend/inpaint/lama/fetch_data/places_standard_test_val_sample.sh

@@ -0,0 +1,22 @@
+mkdir -p places_standard_dataset/val_hires/
+mkdir -p places_standard_dataset/visual_test_hires/
+
+
+# randomly sample images for test and vis
+OUT=$(python3 fetch_data/sampler.py)
+echo ${OUT}
+
+FILELIST=$(cat places_standard_dataset/original/test_random_files.txt)
+
+for i in $FILELIST
+do
+    $(cp ${i} places_standard_dataset/val_hires/)
+done
+
+FILELIST=$(cat places_standard_dataset/original/val_random_files.txt)
+
+for i in $FILELIST
+do
+    $(cp ${i} places_standard_dataset/visual_test_hires/)
+done
+

backend/inpaint/lama/fetch_data/places_standard_train_prepare.sh

@@ -0,0 +1,16 @@
+mkdir -p places_standard_dataset/train
+
+# untar without folder structure
+tar -xvf train_large_places365standard.tar -C places_standard_dataset/train
+
+# create location config places.yaml
+PWD=$(pwd)
+DATASET=${PWD}/places_standard_dataset
+PLACES=${PWD}/configs/training/location/places_standard.yaml
+
+touch $PLACES
+echo "# @package _group_" >> $PLACES
+echo "data_root_dir: ${DATASET}/" >> $PLACES
+echo "out_root_dir: ${PWD}/experiments/" >> $PLACES
+echo "tb_dir: ${PWD}/tb_logs/" >> $PLACES
+echo "pretrained_models: ${PWD}/" >> $PLACES

backend/inpaint/lama/fetch_data/sampler.py

@@ -0,0 +1,42 @@
+import os
+import random
+
+test_files_path           = os.path.abspath('') + '/places_standard_dataset/original/test/'
+list_of_random_test_files = os.path.abspath('') + '/places_standard_dataset/original/test_random_files.txt'
+
+test_files = [
+    test_files_path + image for image in os.listdir(test_files_path)
+]
+
+print(f'Sampling 2000 images out of {len(test_files)} images in {test_files_path}' + \
+      f'and put their paths to {list_of_random_test_files}')
+print('Our training procedure will pick best checkpoints according to metrics, computed on these images.')
+
+random.shuffle(test_files)
+test_files_random = test_files[0:2000]
+with open(list_of_random_test_files, 'w') as fw:
+    for filename in test_files_random:
+        fw.write(filename+'\n')
+print('...done')
+
+
+# --------------------------------
+
+val_files_path           = os.path.abspath('') + '/places_standard_dataset/original/val/'
+list_of_random_val_files = os.path.abspath('') + '/places_standard_dataset/original/val_random_files.txt'
+
+val_files = [
+    val_files_path + image for image in os.listdir(val_files_path)
+]
+
+print(f'Sampling 100 images out of {len(val_files)} in {val_files_path} ' + \
+      f'and put their paths to {list_of_random_val_files}')
+print('We use these images for visual check up of evolution of inpainting algorithm epoch to epoch' )
+
+random.shuffle(val_files)
+val_files_random = val_files[0:100]
+with open(list_of_random_val_files, 'w') as fw:
+    for filename in val_files_random:
+        fw.write(filename+'\n')
+print('...done')      
+