Commited minutiae

minutiae/Dockerfile

@@ -0,0 +1,22 @@
+FROM debian:11
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y wget libgl1 libglib2.0-0 python2 python3 python3-pip libsm6 libxext6 libxrender1
+
+
+# Create main env for Jupyter Notebook
+WORKDIR /tmp
+RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py
+RUN python2 get-pip.py
+RUN python2 -m pip install tensorflow==1.7.0 keras==2.1.6 opencv-python==3.4.8.29 numpy scipy matplotlib pydot graphviz grpcio==1.29.0 markdown==2.6.11 scikit-image
+RUN python3 -m pip install notebook
+RUN python2 -m pip install ipykernel
+RUN python2 -m ipykernel install --user  
+
+# Copy project files
+WORKDIR /src
+COPY ./MinutiaeNet /src/MinutiaeNet
+
+# Launch Jupyter Notebook
+EXPOSE 8888
+CMD ["python3", "-m", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--IdentityProvider.token=''", "--allow-root"]

minutiae/MinutiaeNet/CoarseNet/CoarseNet_model.py

@@ -0,0 +1,947 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+from time import time
+from datetime import datetime
+from CoarseNet_utils import *
+from scipy import misc, ndimage, signal, sparse, io
+import scipy.ndimage
+import cv2
+import sys,os
+sys.path.append(os.path.realpath('../FineNet'))
+from FineNet_model import FineNetmodel
+
+from keras.models import Model
+from keras.layers import Input
+from keras import layers
+from keras.layers.core import Flatten,Activation,Lambda, Dropout
+from keras.layers.convolutional import Conv2D,MaxPooling2D,UpSampling2D,AveragePooling2D
+from keras.layers.normalization import BatchNormalization
+from keras.layers.advanced_activations import PReLU
+from keras.regularizers import l2
+from keras.optimizers import SGD, Adam
+from keras.utils import plot_model
+
+
+import tensorflow as tf
+
+from MinutiaeNet_utils import *
+from LossFunctions import *
+
+
+def conv_bn(bottom, w_size, name, strides=(1,1), dilation_rate=(1,1)):
+    top = Conv2D(w_size[0], (w_size[1],w_size[2]),
+        kernel_regularizer=l2(5e-5),
+        padding='same',
+        strides=strides,
+        dilation_rate=dilation_rate,
+        name='conv-'+name)(bottom)
+    top = BatchNormalization(name='bn-'+name)(top)
+    return top
+
+def conv_bn_prelu(bottom, w_size, name, strides=(1,1), dilation_rate=(1,1)):
+    if dilation_rate == (1,1):
+        conv_type = 'conv'
+    else:
+        conv_type = 'atrousconv'
+
+    top = Conv2D(w_size[0], (w_size[1],w_size[2]),
+        kernel_regularizer=l2(5e-5),
+        padding='same',
+        strides=strides,
+        dilation_rate=dilation_rate,
+        name=conv_type+name)(bottom)
+    top = BatchNormalization(name='bn-'+name)(top)
+    top = PReLU(alpha_initializer='zero', shared_axes=[1,2], name='prelu-'+name)(top)
+    # top = Dropout(0.25)(top)
+    return top
+
+def CoarseNetmodel(input_shape=(400,400,1), weights_path=None, mode='train'):
+    # Change network architecture here!!
+    img_input=Input(input_shape)
+    bn_img=Lambda(img_normalization, name='img_normalized')(img_input)
+
+    # Main part
+    conv = conv_bn_prelu(bn_img, (64, 5, 5), '1_0')
+    conv = conv_bn_prelu(conv, (64, 3, 3), '1_1')
+    conv = conv_bn_prelu(conv, (64, 3, 3), '1_2')
+    conv = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv)
+
+    # =======Block 1 ========
+    conv1 = conv_bn_prelu(conv, (128, 3, 3), '2_1')
+    conv = conv_bn_prelu(conv1, (128, 3, 3), '2_2')
+    conv = conv_bn_prelu(conv, (128, 3, 3), '2_3')
+    conv = layers.add([conv, conv1])
+
+    conv1 = conv_bn_prelu(conv, (128, 3, 3), '2_1b')
+    conv = conv_bn_prelu(conv1, (128, 3, 3), '2_2b')
+    conv = conv_bn_prelu(conv, (128, 3, 3), '2_3b')
+    conv = layers.add([conv, conv1])
+
+    conv1 = conv_bn_prelu(conv, (128, 3, 3), '2_1c')
+    conv = conv_bn_prelu(conv1, (128, 3, 3), '2_2c')
+    conv = conv_bn_prelu(conv, (128, 3, 3), '2_3c')
+    conv = layers.add([conv, conv1])
+
+    conv_block1 = MaxPooling2D(pool_size=(2,2),strides=(2,2))(conv)
+    # ==========================
+
+    # =======Block 2 ========
+    conv1 = conv_bn_prelu(conv_block1, (256,3,3), '3_1')
+    conv = conv_bn_prelu(conv1, (256,3,3), '3_2')
+    conv = conv_bn_prelu(conv, (256,3,3), '3_3')
+    conv = layers.add([conv, conv1])
+
+    conv1 = conv_bn_prelu(conv, (256, 3, 3), '3_1b')
+    conv = conv_bn_prelu(conv1, (256, 3, 3), '3_2b')
+    conv = conv_bn_prelu(conv, (256, 3, 3), '3_3b')
+    conv = layers.add([conv, conv1])
+
+    conv_block2 = MaxPooling2D(pool_size=(2,2),strides=(2,2))(conv)
+    # ==========================
+
+    # =======Block 3 ========
+    conv1 = conv_bn_prelu(conv_block2, (512, 3, 3), '3_1c')
+    conv = conv_bn_prelu(conv1, (512, 3, 3), '3_2c')
+    conv = conv_bn_prelu(conv, (512, 3, 3), '3_3c')
+    conv = layers.add([conv, conv1])
+    conv_block3 = conv_bn_prelu(conv, (256, 3, 3), '3_4c')
+
+    #conv_block3 = MaxPooling2D(pool_size=(2,2),strides=(2,2))(conv)
+    # ==========================
+
+
+    # multi-scale ASPP
+    level_2=conv_bn_prelu(conv_block3, (256,3,3), '4_1', dilation_rate=(1,1))
+    ori_1=conv_bn_prelu(level_2, (128,1,1), 'ori_1_1')
+    ori_1=Conv2D(90, (1,1), padding='same', name='ori_1_2')(ori_1)
+    seg_1=conv_bn_prelu(level_2, (128,1,1), 'seg_1_1')
+    seg_1=Conv2D(1, (1,1), padding='same', name='seg_1_2')(seg_1)
+
+    level_3=conv_bn_prelu(conv_block2, (256,3,3), '4_2', dilation_rate=(4,4))
+    ori_2=conv_bn_prelu(level_3, (128,1,1), 'ori_2_1')
+    ori_2=Conv2D(90, (1,1), padding='same', name='ori_2_2')(ori_2)
+    seg_2=conv_bn_prelu(level_3, (128,1,1), 'seg_2_1')
+    seg_2=Conv2D(1, (1,1), padding='same', name='seg_2_2')(seg_2)
+
+    level_4=conv_bn_prelu(conv_block2, (256,3,3), '4_3', dilation_rate=(8,8))
+    ori_3=conv_bn_prelu(level_4, (128,1,1), 'ori_3_1')
+    ori_3=Conv2D(90, (1,1), padding='same', name='ori_3_2')(ori_3)
+    seg_3=conv_bn_prelu(level_4, (128,1,1), 'seg_3_1')
+    seg_3=Conv2D(1, (1,1), padding='same', name='seg_3_2')(seg_3)
+
+    # sum fusion for ori
+    ori_out=Lambda(merge_sum)([ori_1, ori_2, ori_3])
+    ori_out_1=Activation('sigmoid', name='ori_out_1')(ori_out)
+    ori_out_2=Activation('sigmoid', name='ori_out_2')(ori_out)
+
+    # sum fusion for segmentation
+    seg_out=Lambda(merge_sum)([seg_1, seg_2, seg_3])
+    seg_out=Activation('sigmoid', name='seg_out')(seg_out)
+
+    # ----------------------------------------------------------------------------
+    # enhance part
+    filters_cos, filters_sin = gabor_bank(stride=2, Lambda=8)
+
+    filter_img_real = Conv2D(filters_cos.shape[3],(filters_cos.shape[0],filters_cos.shape[1]),
+        weights=[filters_cos, np.zeros([filters_cos.shape[3]])], padding='same',
+        name='enh_img_real_1')(img_input)
+    filter_img_imag = Conv2D(filters_sin.shape[3],(filters_sin.shape[0],filters_sin.shape[1]),
+        weights=[filters_sin, np.zeros([filters_sin.shape[3]])], padding='same',
+        name='enh_img_imag_1')(img_input)
+
+    ori_peak = Lambda(ori_highest_peak)(ori_out_1)
+    ori_peak = Lambda(select_max)(ori_peak) # select max ori and set it to 1
+
+    # Use this function to upsample image
+    upsample_ori = UpSampling2D(size=(8,8))(ori_peak)
+    seg_round = Activation('softsign')(seg_out)
+
+
+    upsample_seg = UpSampling2D(size=(8,8))(seg_round)
+    mul_mask_real = Lambda(merge_mul)([filter_img_real, upsample_ori])
+
+    enh_img_real = Lambda(reduce_sum, name='enh_img_real_2')(mul_mask_real)
+    mul_mask_imag = Lambda(merge_mul)([filter_img_imag, upsample_ori])
+
+    enh_img_imag = Lambda(reduce_sum, name='enh_img_imag_2')(mul_mask_imag)
+    enh_img = Lambda(atan2, name='phase_img')([enh_img_imag, enh_img_real])
+
+    enh_seg_img = Lambda(merge_concat, name='phase_seg_img')([enh_img, upsample_seg])
+    # ----------------------------------------------------------------------------
+    # mnt part
+    # =======Block 1 ========
+    mnt_conv1 = conv_bn_prelu(enh_seg_img, (64, 9, 9), 'mnt_1_1')
+    mnt_conv = conv_bn_prelu(mnt_conv1, (64, 9, 9), 'mnt_1_2')
+    mnt_conv = conv_bn_prelu(mnt_conv, (64, 9, 9), 'mnt_1_3')
+    mnt_conv = layers.add([mnt_conv, mnt_conv1])
+
+    mnt_conv1 = conv_bn_prelu(mnt_conv, (64, 9, 9), 'mnt_1_1b')
+    mnt_conv = conv_bn_prelu(mnt_conv1, (64, 9, 9), 'mnt_1_2b')
+    mnt_conv = conv_bn_prelu(mnt_conv, (64, 9, 9), 'mnt_1_3b')
+    mnt_conv = layers.add([mnt_conv, mnt_conv1])
+
+    mnt_conv = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(mnt_conv)
+    # ==========================
+
+    # =======Block 2 ========
+    mnt_conv1 = conv_bn_prelu(mnt_conv, (128, 5, 5), 'mnt_2_1')
+    mnt_conv = conv_bn_prelu(mnt_conv1, (128, 5, 5), 'mnt_2_2')
+    mnt_conv = conv_bn_prelu(mnt_conv, (128, 5, 5), 'mnt_2_3')
+    mnt_conv = layers.add([mnt_conv, mnt_conv1])
+
+    mnt_conv1 = conv_bn_prelu(mnt_conv, (128, 5, 5), 'mnt_2_1b')
+    mnt_conv = conv_bn_prelu(mnt_conv1, (128, 5, 5), 'mnt_2_2b')
+    mnt_conv = conv_bn_prelu(mnt_conv, (128, 5, 5), 'mnt_2_3b')
+    mnt_conv = layers.add([mnt_conv, mnt_conv1])
+
+    mnt_conv = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(mnt_conv)
+    # ==========================
+
+    # =======Block 3 ========
+    mnt_conv1 = conv_bn_prelu(mnt_conv, (256, 3, 3), 'mnt_3_1')
+    mnt_conv2 = conv_bn_prelu(mnt_conv1, (256, 3, 3), 'mnt_3_2')
+    mnt_conv3 = conv_bn_prelu(mnt_conv2, (256, 3, 3), 'mnt_3_3')
+    mnt_conv3 = layers.add([mnt_conv3, mnt_conv1])
+    mnt_conv4 = conv_bn_prelu(mnt_conv3, (256, 3, 3), 'mnt_3_4')
+    mnt_conv4 = layers.add([mnt_conv4, mnt_conv2])
+
+    mnt_conv = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(mnt_conv4)
+    # ==========================
+
+
+    mnt_o_1=Lambda(merge_concat)([mnt_conv, ori_out_1])
+    mnt_o_2=conv_bn_prelu(mnt_o_1, (256,1,1), 'mnt_o_1_1')
+    mnt_o_3=Conv2D(180, (1,1), padding='same', name='mnt_o_1_2')(mnt_o_2)
+    mnt_o_out=Activation('sigmoid', name='mnt_o_out')(mnt_o_3)
+
+    mnt_w_1=conv_bn_prelu(mnt_conv, (256,1,1), 'mnt_w_1_1')
+    mnt_w_2=Conv2D(8, (1,1), padding='same', name='mnt_w_1_2')(mnt_w_1)
+    mnt_w_out=Activation('sigmoid', name='mnt_w_out')(mnt_w_2)
+
+    mnt_h_1=conv_bn_prelu(mnt_conv, (256,1,1), 'mnt_h_1_1')
+    mnt_h_2=Conv2D(8, (1,1), padding='same', name='mnt_h_1_2')(mnt_h_1)
+    mnt_h_out=Activation('sigmoid', name='mnt_h_out')(mnt_h_2)
+
+    mnt_s_1=conv_bn_prelu(mnt_conv, (256,1,1), 'mnt_s_1_1')
+    mnt_s_2=Conv2D(1, (1,1), padding='same', name='mnt_s_1_2')(mnt_s_1)
+    mnt_s_out=Activation('sigmoid', name='mnt_s_out')(mnt_s_2)
+
+    if mode == 'deploy':
+        model = Model(inputs=[img_input,], outputs=[enh_img, enh_img_imag, enh_img_real, ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out])
+    else:
+        model = Model(inputs=[img_input,], outputs=[ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out])
+
+    if weights_path != None:
+        model.load_weights(weights_path, by_name=True)
+    return model
+
+def train(input_shape=(400,400), train_set = None,output_dir='../output_CoarseNet/'+datetime.now().strftime('%Y%m%d-%H%M%S'),
+          pretrain_dir=None,batch_size=1,test_set=None, learning_config=None, logging=None):
+
+    img_name, folder_name, img_size = get_maximum_img_size_and_names(train_set, None, max_size=input_shape)
+
+    main_net_model = CoarseNetmodel((img_size[0], img_size[1], 1), pretrain_dir, 'train')
+    # Save model architecture
+    plot_model(main_net_model, to_file=output_dir+'/model.png',show_shapes=True)
+
+    main_net_model.compile(optimizer=learning_config,
+                           loss={'seg_out':segmentation_loss,
+                                 'mnt_o_out': orientation_output_loss, 'mnt_w_out': orientation_output_loss,
+                                 'mnt_h_out': orientation_output_loss, 'mnt_s_out': minutiae_score_loss
+                                 },
+                           loss_weights={'seg_out': .5, 'mnt_w_out': .5, 'mnt_h_out': .5, 'mnt_o_out': 100., 'mnt_s_out': 50.},
+                           metrics={'seg_out':[seg_acc_pos, seg_acc_neg, seg_acc_all],
+                                    'mnt_o_out': [mnt_acc_delta_10, ],
+                                    'mnt_w_out': [mnt_mean_delta, ],
+                                    'mnt_h_out': [mnt_mean_delta, ],
+                                    'mnt_s_out': [seg_acc_pos, seg_acc_neg, seg_acc_all]})
+
+    writer = tf.summary.FileWriter(output_dir)
+
+    Best_F1_result = 0
+    Best_loss = 10000000
+    for epoch in range(1000):
+        outdir = "%s/saved_best_loss/" % (output_dir)
+        mkdir(outdir)
+
+        for i, train in enumerate(load_data((img_name, folder_name, img_size), tra_ori_model, rand=True, aug=0.7, batch_size=batch_size)):
+            loss = main_net_model.train_on_batch(train[0],
+                                                 {'seg_out': train[3],
+                                                  'mnt_w_out': train[4], 'mnt_h_out': train[5], 'mnt_o_out': train[6],
+                                                  'mnt_s_out': train[7]
+                                                  })
+            # Save the lowest loss for easy converge
+            if Best_loss > loss[0]:
+                savedir = "%s%s_%d_%s" % (outdir, str(epoch),i,str(loss[0]))
+                main_net_model.save_weights(savedir, True)
+                Best_loss = loss[0]
+
+            # Write log on screen at every 20 epochs
+            if i%(2/batch_size) == 0:
+                logging.info("epoch=%d, step=%d", epoch, i)
+                # Write details loss
+                logging.info("%s", " ".join(["%s:%.4f\t"%(x) for x in zip(main_net_model.metrics_names, loss)]))
+                # logging.info("Loss = %f    Best loss = %f",loss[0],Best_loss)
+
+                # Show in tensorboard
+                for name, value in zip(main_net_model.metrics_names, loss):
+                    summary = tf.Summary(value=[tf.Summary.Value(tag=name,simple_value=value), ])
+                    writer.add_summary(summary, i)
+
+        # Evaluate every 5 epoch: for faster training
+        if epoch%10 == 0:
+            outdir = "%s/saved_models/" % (output_dir)
+            mkdir(outdir)
+            savedir = "%s%s" % (outdir, str(epoch))
+            main_net_model.save_weights(savedir, True)
+
+            for folder in test_set:
+                precision_test, recall_test, F1_test, precision_test_location, recall_test_location, F1_test_location = evaluate_training(savedir, [folder, ], logging=logging)
+
+            summary = tf.Summary(value=[tf.Summary.Value(tag="Precision", simple_value=precision_test),
+                                        tf.Summary.Value(tag="Recall", simple_value=recall_test),
+                                        tf.Summary.Value(tag="F1", simple_value=F1_test),
+                                        tf.Summary.Value(tag="Location Precision", simple_value=precision_test_location),
+                                        tf.Summary.Value(tag="Location Recall", simple_value=recall_test_location),
+                                        tf.Summary.Value(tag="Location F1", simple_value=F1_test_location), ])
+            writer.add_summary(summary, epoch)
+
+        # Only save the best result
+        if F1_test > Best_F1_result:
+            Best_F1_result = F1_test
+        # else:
+        #     os.remove(savedir)
+
+    writer.close()
+    return
+
+
+def evaluate_training(model_dir, test_set, logging=None, FineNet_path=None):
+    logging.info("Evaluating %s:" % (test_set))
+
+    # Prepare input info
+    img_name, folder_name, img_size = get_maximum_img_size_and_names(test_set)
+
+
+    main_net_model = CoarseNetmodel((None, None, 1), model_dir, 'test')
+
+    ave_prf_nms,ave_prf_nms_location = [],[]
+
+    for j, test in enumerate(
+            load_data((img_name, folder_name, img_size), tra_ori_model, rand=False, aug=0.0, batch_size=1)):
+
+        # logging.info("%d / %d: %s"%(j+1, len(img_name), img_name[j]))
+        ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out = main_net_model.predict(test[0])
+        mnt_gt = label2mnt(test[7], test[4], test[5], test[6])
+
+        original_image = test[0].copy()
+
+        mnt_s_out = mnt_s_out * seg_out
+
+        # Does not useful to use this while training
+        final_minutiae_score_threashold = 0.45
+        early_minutiae_thres = final_minutiae_score_threashold + 0.05
+        isHavingFineNet = False
+
+        # In cases of small amount of minutiae given, try adaptive threshold
+        while final_minutiae_score_threashold >= 0:
+            mnt = label2mnt(mnt_s_out, mnt_w_out, mnt_h_out, mnt_o_out, thresh=early_minutiae_thres)
+            # Previous exp: 0.2
+            mnt_nms_1 = py_cpu_nms(mnt, 0.5)
+            mnt_nms_2 = nms(mnt)
+            # Make sure good result is given
+            if mnt_nms_1.shape[0] > 4 and mnt_nms_2.shape[0] > 4:
+                break
+            else:
+                final_minutiae_score_threashold = final_minutiae_score_threashold - 0.05
+                early_minutiae_thres = early_minutiae_thres - 0.05
+
+        mnt_nms = fuse_nms(mnt_nms_1, mnt_nms_2)
+
+        mnt_nms = mnt_nms[mnt_nms[:, 3] > early_minutiae_thres, :]
+        mnt_refined = []
+        if isHavingFineNet == True:
+            # ======= Verify using FineNet ============
+            patch_minu_radio = 22
+            if FineNet_path != None:
+                for idx_minu in range(mnt_nms.shape[0]):
+                    try:
+                        # Extract patch from image
+                        x_begin = int(mnt_nms[idx_minu, 1]) - patch_minu_radio
+                        y_begin = int(mnt_nms[idx_minu, 0]) - patch_minu_radio
+                        patch_minu = original_image[x_begin:x_begin + 2 * patch_minu_radio,
+                                     y_begin:y_begin + 2 * patch_minu_radio]
+
+                        patch_minu = cv2.resize(patch_minu, dsize=(224, 224), interpolation=cv2.INTER_NEAREST)
+
+                        ret = np.empty((patch_minu.shape[0], patch_minu.shape[1], 3), dtype=np.uint8)
+                        ret[:, :, 0] = patch_minu
+                        ret[:, :, 1] = patch_minu
+                        ret[:, :, 2] = patch_minu
+                        patch_minu = ret
+                        patch_minu = np.expand_dims(patch_minu, axis=0)
+
+                        # # Can use class as hard decision
+                        # # 0: minu  1: non-minu
+                        # [class_Minutiae] = np.argmax(model_FineNet.predict(patch_minu), axis=1)
+                        #
+                        # if class_Minutiae == 0:
+                        #     mnt_refined.append(mnt_nms[idx_minu,:])
+
+                        # Use soft decision: merge FineNet score with CoarseNet score
+                        [isMinutiaeProb] = model_FineNet.predict(patch_minu)
+                        isMinutiaeProb = isMinutiaeProb[0]
+                        # print isMinutiaeProb
+                        tmp_mnt = mnt_nms[idx_minu, :].copy()
+                        tmp_mnt[3] = (4 * tmp_mnt[3] + isMinutiaeProb) / 5
+                        mnt_refined.append(tmp_mnt)
+
+                    except:
+                        mnt_refined.append(mnt_nms[idx_minu, :])
+        else:
+            mnt_refined = mnt_nms
+
+        mnt_nms = np.array(mnt_refined)
+
+        if mnt_nms.shape[0] > 0:
+            mnt_nms = mnt_nms[mnt_nms[:, 3] > final_minutiae_score_threashold, :]
+
+        p, r, f, l, o = metric_P_R_F(mnt_gt, mnt_nms, 16, np.pi/6)
+        ave_prf_nms.append([p, r, f, l, o])
+        p, r, f, l, o = metric_P_R_F(mnt_gt, mnt_nms, 16, np.pi)
+        ave_prf_nms_location.append([p, r, f, l, o])
+
+
+    logging.info("Average testing results:")
+    ave_prf_nms = np.mean(np.array(ave_prf_nms), 0)
+    ave_prf_nms_location = np.mean(np.array(ave_prf_nms_location), 0)
+    logging.info(
+        "Precision: %f\tRecall: %f\tF1-measure: %f\tLocation_dis: %f\tOrientation_delta:%f\n----------------\n" % (
+            ave_prf_nms[0], ave_prf_nms[1], ave_prf_nms[2], ave_prf_nms[3], ave_prf_nms[4]))
+
+    return ave_prf_nms[0], ave_prf_nms[1], ave_prf_nms[2], ave_prf_nms_location[0], ave_prf_nms_location[1], ave_prf_nms_location[2]
+
+def fuse_minu_orientation(dir_map, mnt, mode=1,block_size=16):
+    # mode is the way to fuse output minutiae with orientation
+    # 1: use orientation; 2: use minutiae; 3: fuse average
+    blkH, blkW = dir_map.shape
+    dir_map = dir_map%(2*np.pi)
+
+    if mode == 1:
+        for k in range(mnt.shape[0]):
+            # Choose nearest orientation
+            ori_value = dir_map[int(mnt[k, 1]//block_size),int(mnt[k, 0]//block_size)]
+            if 0 < mnt[k, 2] and mnt[k, 2] <= np.pi/2:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    mnt[k, 2] = ori_value
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    if (ori_value - mnt[k, 2]) < (np.pi - ori_value + mnt[k, 2]):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value + np.pi
+                if np.pi < ori_value and ori_value <= 3*np.pi/2:
+                    mnt[k, 2] = ori_value - np.pi
+                if 3*np.pi/2 < ori_value and ori_value <= 2 * np.pi:
+                    if (np.pi*2 - ori_value + mnt[k, 2]) < (ori_value - np.pi - mnt[k, 2]):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value - np.pi
+            if np.pi/2 < mnt[k, 2] and mnt[k, 2] <= np.pi:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    if (mnt[k, 2] - ori_value) < (np.pi - ori_value + mnt[k, 2]):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value + np.pi
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    mnt[k, 2] = ori_value
+                if np.pi < ori_value and ori_value <= 3*np.pi/2:
+                    if (ori_value - mnt[k, 2]) < (mnt[k, 2] - ori_value + np.pi):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value - np.pi
+                if 3*np.pi/2 < ori_value and ori_value <= 2 * np.pi:
+                    mnt[k, 2] = ori_value - np.pi
+            if np.pi < mnt[k, 2] and mnt[k, 2] <= 3*np.pi/2:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    mnt[k, 2] = ori_value + np.pi
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    if (mnt[k, 2] - ori_value) < (ori_value + np.pi - mnt[k, 2]):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value + np.pi
+                if np.pi < ori_value and ori_value <= 3*np.pi/2:
+                    mnt[k, 2] = ori_value
+                if 3*np.pi/2 < ori_value and ori_value <= 2 * np.pi:
+                    if (ori_value - mnt[k, 2]) < (mnt[k, 2] - ori_value + np.pi):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value - np.pi
+            if 3*np.pi/2 < mnt[k, 2] and mnt[k, 2] <= 2*np.pi:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    if (np.pi - mnt[k, 2] + ori_value) < (mnt[k, 2] - np.pi - ori_value):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value + np.pi
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    mnt[k, 2] = ori_value + np.pi
+                if np.pi < ori_value and ori_value <= 3*np.pi/2:
+                    if (mnt[k, 2] - ori_value) < (np.pi*2 - mnt[k, 2] + ori_value - np.pi):
+                        mnt[k, 2] = ori_value
+                    else:
+                        mnt[k, 2] = ori_value - np.pi
+                if 3*np.pi/2 < ori_value and ori_value <= 2 * np.pi:
+                    mnt[k, 2] = ori_value
+
+
+    elif mode == 2:
+        return
+    elif mode ==3:
+        for k in range(mnt.shape[0]):
+            # Choose nearest orientation
+
+            ori_value = dir_map[int(mnt[k, 1] // block_size), int(mnt[k, 0] // block_size)]
+            if 0 < mnt[k, 2] and mnt[k, 2] <= np.pi / 2:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    fixed_ori = ori_value
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    if (ori_value - mnt[k, 2]) < (np.pi - ori_value + mnt[k, 2]):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value + np.pi
+                if np.pi < ori_value and ori_value <= 3 * np.pi / 2:
+                    fixed_ori = ori_value - np.pi
+                if 3 * np.pi / 2 < ori_value and ori_value <= 2 * np.pi:
+                    if (np.pi * 2 - ori_value + mnt[k, 2]) < (ori_value - np.pi - mnt[k, 2]):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value - np.pi
+            if np.pi / 2 < mnt[k, 2] and mnt[k, 2] <= np.pi:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    if (mnt[k, 2] - ori_value) < (np.pi - ori_value + mnt[k, 2]):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value + np.pi
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    fixed_ori = ori_value
+                if np.pi < ori_value and ori_value <= 3 * np.pi / 2:
+                    if (ori_value - mnt[k, 2]) < (mnt[k, 2] - ori_value + np.pi):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value - np.pi
+                if 3 * np.pi / 2 < ori_value and ori_value <= 2 * np.pi:
+                    fixed_ori = ori_value - np.pi
+            if np.pi < mnt[k, 2] and mnt[k, 2] <= 3 * np.pi / 2:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    fixed_ori = ori_value + np.pi
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    if (mnt[k, 2] - ori_value) < (ori_value + np.pi - mnt[k, 2]):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value + np.pi
+                if np.pi < ori_value and ori_value <= 3 * np.pi / 2:
+                    fixed_ori = ori_value
+                if 3 * np.pi / 2 < ori_value and ori_value <= 2 * np.pi:
+                    if (ori_value - mnt[k, 2]) < (mnt[k, 2] - ori_value + np.pi):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value - np.pi
+            if 3 * np.pi / 2 < mnt[k, 2] and mnt[k, 2] <= 2 * np.pi:
+                if 0 < ori_value and ori_value <= np.pi / 2:
+                    if (np.pi - mnt[k, 2] + ori_value) < (mnt[k, 2] - np.pi - ori_value):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value + np.pi
+                if np.pi / 2 < ori_value and ori_value <= np.pi:
+                    fixed_ori = ori_value + np.pi
+                if np.pi < ori_value and ori_value <= 3 * np.pi / 2:
+                    if (mnt[k, 2] - ori_value) < (np.pi * 2 - mnt[k, 2] + ori_value - np.pi):
+                        fixed_ori = ori_value
+                    else:
+                        fixed_ori = ori_value - np.pi
+                if 3 * np.pi / 2 < ori_value and ori_value <= 2 * np.pi:
+                    fixed_ori = ori_value
+
+            mnt[k, 2] = (mnt[k, 2] + fixed_ori)/2.0
+    else:
+        return
+
+def deploy_with_GT(deploy_set, output_dir, model_path, FineNet_path=None, set_name=None):
+    if set_name is None:
+        set_name = deploy_set.split('/')[-2]
+
+    # Read image and GT
+    img_name, folder_name, img_size = get_maximum_img_size_and_names(deploy_set)
+
+    mkdir(output_dir + '/'+ set_name + '/')
+    mkdir(output_dir + '/' + set_name + '/mnt_results/')
+    mkdir(output_dir + '/'+ set_name + '/seg_results/')
+    mkdir(output_dir + '/' + set_name + '/OF_results/')
+
+    logging.info("Predicting %s:" % (set_name))
+
+    isHavingFineNet = False
+
+    main_net_model = CoarseNetmodel((None, None, 1), model_path, mode='deploy')
+
+    if isHavingFineNet == True:
+        # ====== Load FineNet to verify
+        model_FineNet = FineNetmodel(num_classes=2,
+                             pretrained_path=FineNet_path,
+                             input_shape=(224,224,3))
+
+        model_FineNet.compile(loss='categorical_crossentropy',
+                      optimizer=Adam(lr=0),
+                      metrics=['accuracy'])
+
+    time_c = []
+    ave_prf_nms=[]
+    for i, test in enumerate(
+            load_data((img_name, folder_name, img_size), tra_ori_model, rand=False, aug=0.0, batch_size=1)):
+
+        print i, img_name[i]
+        logging.info("%s %d / %d: %s" % (set_name, i + 1, len(img_name), img_name[i]))
+        time_start = time()
+
+        image = misc.imread(deploy_set + 'img_files/' + img_name[i] + '.bmp', mode='L')# / 255.0
+        mask = misc.imread(deploy_set + 'seg_files/' + img_name[i] + '.bmp', mode='L') / 255.0
+
+        img_size = image.shape
+        img_size = np.array(img_size, dtype=np.int32) // 8 * 8
+        image = image[:img_size[0], :img_size[1]]
+        mask = mask[:img_size[0], :img_size[1]]
+
+
+        original_image = image.copy()
+
+
+        # Generate OF
+        texture_img = FastEnhanceTexture(image, sigma=2.5, show=False)
+        dir_map, fre_map = get_maps_STFT(texture_img, patch_size=64, block_size=16, preprocess=True)
+
+        image = np.reshape(image, [1, image.shape[0], image.shape[1], 1])
+
+        enh_img, enh_img_imag, enhance_img, ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out \
+            = main_net_model.predict(image)
+
+        time_afterconv = time()
+
+        # Use post processing to smooth image
+        round_seg = np.round(np.squeeze(seg_out))
+        seg_out = 1 - round_seg
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
+        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_CLOSE, kernel)
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7))
+        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_OPEN, kernel)
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
+        seg_out = cv2.dilate(seg_out, kernel)
+
+        # If use mask from outside
+        # seg_out = cv2.resize(mask, dsize=(seg_out.shape[1], seg_out.shape[0]))
+
+        mnt_gt = label2mnt(test[7], test[4], test[5], test[6])
+
+        final_minutiae_score_threashold = 0.45
+        early_minutiae_thres = final_minutiae_score_threashold + 0.05
+
+
+        # In cases of small amount of minutiae given, try adaptive threshold
+        while final_minutiae_score_threashold >= 0:
+            mnt = label2mnt(np.squeeze(mnt_s_out) * np.round(np.squeeze(seg_out)), mnt_w_out, mnt_h_out, mnt_o_out,
+                            thresh=early_minutiae_thres)
+
+            # Previous exp: 0.2
+            mnt_nms_1 = py_cpu_nms(mnt, 0.5)
+            mnt_nms_2 = nms(mnt)
+            # Make sure good result is given
+            if mnt_nms_1.shape[0] > 4 and mnt_nms_2.shape[0] > 4:
+                break
+            else:
+                final_minutiae_score_threashold = final_minutiae_score_threashold - 0.05
+                early_minutiae_thres = early_minutiae_thres - 0.05
+
+
+        mnt_nms = fuse_nms(mnt_nms_1, mnt_nms_2)
+
+        mnt_nms = mnt_nms[mnt_nms[:, 3] > early_minutiae_thres, :]
+        mnt_refined = []
+        if isHavingFineNet == True:
+            # ======= Verify using FineNet ============
+            patch_minu_radio = 22
+            if FineNet_path != None:
+                for idx_minu in range(mnt_nms.shape[0]):
+                    try:
+                        # Extract patch from image
+                        x_begin = int(mnt_nms[idx_minu, 1]) - patch_minu_radio
+                        y_begin = int(mnt_nms[idx_minu, 0]) - patch_minu_radio
+                        patch_minu = original_image[x_begin:x_begin + 2 * patch_minu_radio,
+                                     y_begin:y_begin + 2 * patch_minu_radio]
+
+                        patch_minu = cv2.resize(patch_minu, dsize=(224, 224), interpolation=cv2.INTER_NEAREST)
+
+                        ret = np.empty((patch_minu.shape[0], patch_minu.shape[1], 3), dtype=np.uint8)
+                        ret[:, :, 0] = patch_minu
+                        ret[:, :, 1] = patch_minu
+                        ret[:, :, 2] = patch_minu
+                        patch_minu = ret
+                        patch_minu = np.expand_dims(patch_minu, axis=0)
+
+                        # # Can use class as hard decision
+                        # # 0: minu  1: non-minu
+                        # [class_Minutiae] = np.argmax(model_FineNet.predict(patch_minu), axis=1)
+                        #
+                        # if class_Minutiae == 0:
+                        #     mnt_refined.append(mnt_nms[idx_minu,:])
+
+                        # Use soft decision: merge FineNet score with CoarseNet score
+                        [isMinutiaeProb] = model_FineNet.predict(patch_minu)
+                        isMinutiaeProb = isMinutiaeProb[0]
+                        # print isMinutiaeProb
+                        tmp_mnt = mnt_nms[idx_minu, :].copy()
+                        tmp_mnt[3] = (4*tmp_mnt[3] + isMinutiaeProb) / 5
+                        mnt_refined.append(tmp_mnt)
+
+                    except:
+                        mnt_refined.append(mnt_nms[idx_minu, :])
+        else:
+            mnt_refined = mnt_nms
+
+        mnt_nms = np.array(mnt_refined)
+
+        if mnt_nms.shape[0] > 0:
+            mnt_nms = mnt_nms[mnt_nms[:, 3] > final_minutiae_score_threashold, :]
+
+        final_mask = ndimage.zoom(np.round(np.squeeze(seg_out)), [8, 8], order=0)
+
+        # Show the orientation
+        show_orientation_field(original_image, dir_map + np.pi, mask=final_mask,
+                               fname="%s/%s/OF_results/%s_OF.jpg" % (output_dir, set_name, img_name[i]))
+
+        fuse_minu_orientation(dir_map, mnt_nms, mode=3)
+
+        time_afterpost = time()
+        mnt_writer(mnt_nms, img_name[i], img_size, "%s/%s/mnt_results/%s.mnt" % (output_dir, set_name, img_name[i]))
+        draw_minutiae_overlay_with_score(image, mnt_nms, mnt_gt[:, :3], "%s/%s/%s_minu.jpg"%(output_dir, set_name, img_name[i]),saveimage=True)
+        # misc.imsave("%s/%s/%s_score.jpg"%(output_dir, set_name, img_name[i]), np.squeeze(mnt_s_out_upscale))
+
+        misc.imsave("%s/%s/seg_results/%s_seg.jpg" % (output_dir, set_name, img_name[i]), final_mask)
+
+        time_afterdraw = time()
+        time_c.append([time_afterconv - time_start, time_afterpost - time_afterconv, time_afterdraw - time_afterpost])
+        logging.info(
+            "load+conv: %.3fs, seg-postpro+nms: %.3f, draw: %.3f" % (time_c[-1][0], time_c[-1][1], time_c[-1][2]))
+
+        # Metrics calculating
+        p, r, f, l, o = metric_P_R_F(mnt_gt, mnt_nms)
+        ave_prf_nms.append([p, r, f, l, o])
+        print p,r,f
+
+    time_c = np.mean(np.array(time_c), axis=0)
+    ave_prf_nms = np.mean(np.array(ave_prf_nms), 0)
+    print "Precision: %f\tRecall: %f\tF1-measure: %f" % (ave_prf_nms[0], ave_prf_nms[1], ave_prf_nms[2])
+
+    logging.info(
+        "Average: load+conv: %.3fs, oir-select+seg-post+nms: %.3f, draw: %.3f" % (time_c[0], time_c[1], time_c[2]))
+    return
+
+def inference(deploy_set, output_dir, model_path, FineNet_path=None, set_name=None, file_ext='.bmp', isHavingFineNet = False):
+    if set_name is None:
+        set_name = deploy_set.split('/')[-2]
+
+
+    mkdir(output_dir + '/'+ set_name + '/')
+    mkdir(output_dir + '/' + set_name + '/mnt_results/')
+    mkdir(output_dir + '/'+ set_name + '/seg_results/')
+    mkdir(output_dir + '/' + set_name + '/OF_results/')
+
+    logging.info("Predicting %s:" % (set_name))
+
+    _, img_name = get_files_in_folder(deploy_set+ 'img_files/', file_ext)
+    print deploy_set
+
+    # ====== Load FineNet to verify
+    if isHavingFineNet == True:
+        model_FineNet = FineNetmodel(num_classes=2,
+                             pretrained_path=FineNet_path,
+                             input_shape=(224,224,3))
+
+        model_FineNet.compile(loss='categorical_crossentropy',
+                      optimizer=Adam(lr=0),
+                      metrics=['accuracy'])
+
+    time_c = []
+
+    main_net_model = CoarseNetmodel((None, None, 1), model_path, mode='deploy')
+
+    for i in xrange(0, len(img_name)):
+        print i
+
+        image = misc.imread(deploy_set + 'img_files/'+ img_name[i] + file_ext, mode='L')  # / 255.0
+
+        img_size = image.shape
+        img_size = np.array(img_size, dtype=np.int32) // 8 * 8
+
+        # read the mask from files
+        try:
+            mask = misc.imread(deploy_set + 'seg_files/' + img_name[i] + '.jpg', mode='L') / 255.0
+        except:
+            mask = np.ones((img_size[0],img_size[1]))
+
+
+        image = image[:img_size[0], :img_size[1]]
+        mask = mask[:img_size[0], :img_size[1]]
+
+        original_image = image.copy()
+
+        texture_img = FastEnhanceTexture(image, sigma=2.5, show=False)
+        dir_map, fre_map = get_maps_STFT(texture_img, patch_size=64, block_size=16, preprocess=True)
+
+        image = image*mask
+
+        logging.info("%s %d / %d: %s" % (set_name, i + 1, len(img_name), img_name[i]))
+        time_start = time()
+
+        image = np.reshape(image, [1, image.shape[0], image.shape[1], 1])
+
+        enh_img, enh_img_imag, enhance_img, ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out \
+            = main_net_model.predict(image)
+        time_afterconv = time()
+
+        # If use mask from model
+        round_seg = np.round(np.squeeze(seg_out))
+        seg_out = 1 - round_seg
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))
+        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_CLOSE, kernel)
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7))
+        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_OPEN, kernel)
+        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
+        seg_out = cv2.dilate(seg_out, kernel)
+
+        # If use mask from outside
+        # seg_out = cv2.resize(mask, dsize=(seg_out.shape[1], seg_out.shape[0]))
+
+
+        max_num_minu = 20
+        min_num_minu = 6
+
+        early_minutiae_thres = 0.5
+
+
+        # New adaptive threshold
+        mnt = label2mnt(np.squeeze(mnt_s_out) * np.round(np.squeeze(seg_out)), mnt_w_out, mnt_h_out, mnt_o_out,
+                        thresh=0)
+
+        # Previous exp: 0.2
+        mnt_nms_1 = py_cpu_nms(mnt, 0.5)
+        mnt_nms_2 = nms(mnt)
+        mnt_nms_1.view('f8,f8,f8,f8').sort(order=['f3'], axis=0)
+        mnt_nms_1 = mnt_nms_1[::-1]
+
+        mnt_nms_1_copy = mnt_nms_1.copy()
+        mnt_nms_2_copy = mnt_nms_2.copy()
+        # Adaptive threshold goes here
+        # Make sure the maximum number of minutiae is max_num_minu
+
+        # Sort minutiae by score
+        while early_minutiae_thres > 0:
+            mnt_nms_1 = mnt_nms_1_copy[mnt_nms_1_copy[:, 3] > early_minutiae_thres, :]
+            mnt_nms_2 = mnt_nms_2_copy[mnt_nms_2_copy[:, 3] > early_minutiae_thres, :]
+
+            if mnt_nms_1.shape[0]>max_num_minu or mnt_nms_2.shape[0]>max_num_minu:
+                mnt_nms_1 = mnt_nms_1[:max_num_minu,:]
+                mnt_nms_2 = mnt_nms_2[:max_num_minu, :]
+            if mnt_nms_1.shape[0] > min_num_minu and mnt_nms_2.shape[0] > min_num_minu:
+                break
+
+            early_minutiae_thres = early_minutiae_thres - 0.05
+
+        mnt_nms = fuse_nms(mnt_nms_1, mnt_nms_2)
+
+        final_minutiae_score_threashold = early_minutiae_thres - 0.05
+
+        print early_minutiae_thres, final_minutiae_score_threashold
+
+        mnt_refined = []
+        if isHavingFineNet == True:
+            # ======= Verify using FineNet ============
+            patch_minu_radio = 22
+            if FineNet_path != None:
+                for idx_minu in range(mnt_nms.shape[0]):
+                    try:
+                        # Extract patch from image
+                        x_begin = int(mnt_nms[idx_minu, 1]) - patch_minu_radio
+                        y_begin = int(mnt_nms[idx_minu, 0]) - patch_minu_radio
+                        patch_minu = original_image[x_begin:x_begin + 2 * patch_minu_radio,
+                                                    y_begin:y_begin + 2 * patch_minu_radio]
+
+                        patch_minu = cv2.resize(patch_minu, dsize=(224, 224),interpolation=cv2.INTER_NEAREST)
+
+                        ret = np.empty((patch_minu.shape[0], patch_minu.shape[1], 3), dtype=np.uint8)
+                        ret[:, :, 0] = patch_minu
+                        ret[:, :, 1] = patch_minu
+                        ret[:, :, 2] = patch_minu
+                        patch_minu = ret
+                        patch_minu = np.expand_dims(patch_minu, axis=0)
+
+                        # # Can use class as hard decision
+                        # # 0: minu  1: non-minu
+                        # [class_Minutiae] = np.argmax(model_FineNet.predict(patch_minu), axis=1)
+                        #
+                        # if class_Minutiae == 0:
+                        #     mnt_refined.append(mnt_nms[idx_minu,:])
+
+
+                        # Use soft decision: merge FineNet score with CoarseNet score
+                        [isMinutiaeProb] = model_FineNet.predict(patch_minu)
+                        isMinutiaeProb = isMinutiaeProb[0]
+                        #print isMinutiaeProb
+                        tmp_mnt = mnt_nms[idx_minu, :].copy()
+                        tmp_mnt[3] = (4*tmp_mnt[3] + isMinutiaeProb)/5
+                        mnt_refined.append(tmp_mnt)
+
+                    except:
+                        mnt_refined.append(mnt_nms[idx_minu, :])
+        else:
+            mnt_refined = mnt_nms
+
+        mnt_nms_backup = mnt_nms.copy()
+        mnt_nms = np.array(mnt_refined)
+
+        if mnt_nms.shape[0] > 0:
+            mnt_nms = mnt_nms[mnt_nms[:,3]>final_minutiae_score_threashold,:]
+
+        final_mask = ndimage.zoom(np.round(np.squeeze(seg_out)), [8, 8], order=0)
+        # Show the orientation
+        show_orientation_field(original_image, dir_map + np.pi, mask=final_mask, fname="%s/%s/OF_results/%s_OF.jpg" % (output_dir, set_name, img_name[i]))
+
+        fuse_minu_orientation(dir_map, mnt_nms, mode=3)
+
+        time_afterpost = time()
+        mnt_writer(mnt_nms, img_name[i], img_size, "%s/%s/mnt_results/%s.mnt"%(output_dir, set_name, img_name[i]))
+        draw_minutiae(original_image, mnt_nms, "%s/%s/%s_minu.jpg"%(output_dir, set_name, img_name[i]),saveimage=True)
+
+        misc.imsave("%s/%s/seg_results/%s_seg.jpg" % (output_dir, set_name, img_name[i]), final_mask)
+
+        time_afterdraw = time()
+        time_c.append([time_afterconv - time_start, time_afterpost - time_afterconv, time_afterdraw - time_afterpost])
+        logging.info(
+            "load+conv: %.3fs, seg-postpro+nms: %.3f, draw: %.3f" % (time_c[-1][0], time_c[-1][1], time_c[-1][2]))
+
+
+    # time_c = np.mean(np.array(time_c), axis=0)
+    # logging.info(
+    #     "Average: load+conv: %.3fs, oir-select+seg-post+nms: %.3f, draw: %.3f" % (time_c[0], time_c[1], time_c[2]))
+    return

minutiae/MinutiaeNet/CoarseNet/CoarseNet_run.py

@@ -0,0 +1,65 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+import os
+os.environ['KERAS_BACKEND'] = 'tensorflow'
+
+from datetime import datetime
+from keras import backend as K
+
+from MinutiaeNet_utils import *
+from CoarseNet_utils import *
+from CoarseNet_model import *
+
+
+
+os.environ["CUDA_VISIBLE_DEVICES"] = '0'
+
+config = K.tf.ConfigProto(gpu_options=K.tf.GPUOptions(allow_growth=True))
+sess = K.tf.Session(config=config)
+K.set_session(sess)
+
+# mode = 'inference'
+mode = 'deploy'
+
+# Can use multiple folders for deploy, inference
+deploy_set = ['../Dataset/CoarseNet_train/',]
+inference_set = ['../Dataset/CoarseNet_test/',]
+
+
+pretrain_dir = '../Models/CoarseNet.h5'
+output_dir = '../output_CoarseNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')
+
+FineNet_dir = '../Models/FineNet.h5'
+
+def main():
+    if mode == 'deploy':
+        output_dir = '../output_CoarseNet/deployResults/' +datetime.now().strftime('%Y%m%d-%H%M%S')
+        logging = init_log(output_dir)
+        for i, folder in enumerate(deploy_set):
+            deploy_with_GT(folder, output_dir=output_dir, model_path=pretrain_dir, FineNet_path=FineNet_dir)
+            # evaluate_training(model_dir=pretrain_dir, test_set=folder, logging=logging)
+    elif mode == 'inference':
+        output_dir = '../output_CoarseNet/inferenceResults/' +datetime.now().strftime('%Y%m%d-%H%M%S')
+        logging = init_log(output_dir)
+        for i, folder in enumerate(inference_set):
+            inference(folder, output_dir=output_dir, model_path=pretrain_dir, FineNet_path=FineNet_dir, file_ext='.bmp',
+                      isHavingFineNet=False)
+    else:
+        pass
+
+if __name__ =='__main__':
+    main()

minutiae/MinutiaeNet/CoarseNet/CoarseNet_train.py

@@ -0,0 +1,69 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+import os
+os.environ['KERAS_BACKEND'] = 'tensorflow'
+
+from datetime import datetime
+from MinutiaeNet_utils import *
+
+from keras import backend as K
+from keras.optimizers import SGD, Adam
+
+from CoarseNet_utils import *
+from CoarseNet_model import *
+import argparse
+
+parser = argparse.ArgumentParser(description='Minutiae Net')
+parser.add_argument('lr', type=str, default="0.005",
+                    help='Setting learning rate')
+
+parser.add_argument('GPU', type=str, default="0",
+                    help='Choosing GPU')
+
+args = parser.parse_args()
+
+
+os.environ["CUDA_VISIBLE_DEVICES"] = args.GPU
+
+config = K.tf.ConfigProto(gpu_options=K.tf.GPUOptions(allow_growth=True))
+sess = K.tf.Session(config=config)
+K.set_session(sess)
+
+batch_size = 2
+use_multiprocessing = False
+input_size = 400
+
+# Can use multiple folders for training
+train_set = ['../Dataset/CoarseNet_train/',]
+
+validate_set = ['../path/to/your/data/',]
+
+pretrain_dir = '../Models/CoarseNet.h5'
+output_dir = '../output_CoarseNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')
+FineNet_dir = '../Models/FineNet.h5'
+
+if __name__ =='__main__':
+
+    output_dir = '../output_CoarseNet/trainResults/' + datetime.now().strftime('%Y%m%d-%H%M%S')
+    logging = init_log(output_dir)
+    logging.info("Learning rate = %s", args.lr)
+    logging.info("Pretrain dir = %s", pretrain_dir)
+
+    train(input_shape=(input_size, input_size), train_set=train_set, output_dir=output_dir,
+          pretrain_dir=pretrain_dir, batch_size=batch_size, test_set=validate_set,
+          learning_config=Adam(lr=float(args.lr), beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=0.9),
+          logging=logging)

minutiae/MinutiaeNet/CoarseNet/CoarseNet_utils.py

@@ -0,0 +1,386 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+
+from functools import partial
+from multiprocessing import Pool
+from MinutiaeNet_utils import *
+from scipy import misc, ndimage, signal, sparse
+import numpy as np
+
+from keras import backend as K
+from keras.models import Model
+from keras.layers import Input
+from keras.layers.core import Lambda
+import tensorflow as tf
+
+def sub_load_data(data, img_size, aug):
+    img_name, dataset = data
+
+    img = misc.imread(dataset+'img_files/'+img_name+'.bmp', mode='L')
+
+
+    try:
+        seg = misc.imread(dataset + 'seg_files/' + img_name + '.bmp', mode='L')
+    except:
+        seg = np.ones_like(img)
+
+    try:
+        ali = misc.imread(dataset+'ori_files/'+img_name+'.jpg', mode='L')
+    except:
+        ali = np.zeros_like(img)
+    mnt = np.array(mnt_reader(dataset+'mnt_files/'+img_name+'.mnt'), dtype=float)
+
+    if any(img.shape != img_size):
+        # random pad mean values to reach required shape
+        if np.random.rand()<aug:
+            tra = np.int32(np.random.rand(2)*(np.array(img_size)-np.array(img.shape)))
+        else:
+            tra = np.int32(0.5*(np.array(img_size)-np.array(img.shape)))
+
+        img_t = np.ones(img_size)*np.mean(img)
+        seg_t = np.zeros(img_size)
+        ali_t = np.ones(img_size)*np.mean(ali)
+
+        img_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = img
+        seg_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = seg
+        ali_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = ali
+
+        img = img_t
+        seg = seg_t
+        ali = ali_t
+        mnt = mnt+np.array([tra[1],tra[0],0])
+
+    if np.random.rand()<aug:
+        # random rotation [0 - 360] & translation img_size / 4
+        rot = np.random.rand() * 360
+        tra = (np.random.rand(2)-0.5) / 2 * img_size
+        img = ndimage.rotate(img, rot, reshape=False, mode='reflect')
+        img = ndimage.shift(img, tra, mode='reflect')
+        seg = ndimage.rotate(seg, rot, reshape=False, mode='constant')
+        seg = ndimage.shift(seg, tra, mode='constant')
+        ali = ndimage.rotate(ali, rot, reshape=False, mode='reflect')
+        ali = ndimage.shift(ali, tra, mode='reflect')
+        mnt_r = point_rot(mnt[:, :2], rot/180*np.pi, img.shape, img.shape)
+        mnt = np.column_stack((mnt_r+tra[[1, 0]], mnt[:, 2]-rot/180*np.pi))
+
+    # only keep mnt that stay in pic & not on border
+    mnt = mnt[(8<=mnt[:,0])*(mnt[:,0]<img_size[1]-8)*(8<=mnt[:, 1])*(mnt[:,1]<img_size[0]-8), :]
+    return img, seg, ali, mnt
+
+use_multiprocessing = False
+def load_data(dataset, tra_ori_model, rand=False, aug=0.0, batch_size=1, sample_rate=None):
+
+    if type(dataset[0]) == str:
+        img_name, folder_name, img_size = get_maximum_img_size_and_names(dataset, sample_rate)
+    else:
+        img_name, folder_name, img_size = dataset
+
+    if rand:
+        rand_idx = np.arange(len(img_name))
+        np.random.shuffle(rand_idx)
+        img_name = img_name[rand_idx]
+        folder_name = folder_name[rand_idx]
+
+    if batch_size > 1 and use_multiprocessing==True:
+        p = Pool(batch_size)
+
+    p_sub_load_data = partial(sub_load_data, img_size=img_size, aug=aug)
+
+    for i in xrange(0,len(img_name), batch_size):
+        have_alignment = np.ones([batch_size, 1, 1, 1])
+        image = np.zeros((batch_size, img_size[0], img_size[1], 1))
+        segment = np.zeros((batch_size, img_size[0], img_size[1], 1))
+        alignment = np.zeros((batch_size, img_size[0], img_size[1], 1))
+
+        minutiae_w = np.zeros((batch_size, img_size[0]/8, img_size[1]/8, 1))-1
+        minutiae_h = np.zeros((batch_size, img_size[0]/8, img_size[1]/8, 1))-1
+        minutiae_o = np.zeros((batch_size, img_size[0]/8, img_size[1]/8, 1))-1
+
+        batch_name = [img_name[(i+j)%len(img_name)] for j in xrange(batch_size)]
+        batch_f_name = [folder_name[(i+j)%len(img_name)] for j in xrange(batch_size)]
+
+        if batch_size > 1 and use_multiprocessing==True:
+            results = p.map(p_sub_load_data, zip(batch_name, batch_f_name))
+        else:
+            results = map(p_sub_load_data, zip(batch_name, batch_f_name))
+
+        for j in xrange(batch_size):
+            img, seg, ali, mnt = results[j]
+            if np.sum(ali) == 0:
+                have_alignment[j, 0, 0, 0] = 0
+            image[j, :, :, 0] = img / 255.0
+            segment[j, :, :, 0] = seg / 255.0
+            alignment[j, :, :, 0] = ali / 255.0
+            minutiae_w[j, (mnt[:, 1]/8).astype(int), (mnt[:, 0]/8).astype(int), 0] = mnt[:, 0] % 8
+            minutiae_h[j, (mnt[:, 1]/8).astype(int), (mnt[:, 0]/8).astype(int), 0] = mnt[:, 1] % 8
+            minutiae_o[j, (mnt[:, 1]/8).astype(int), (mnt[:, 0]/8).astype(int), 0] = mnt[:, 2]
+
+        # get seg
+        label_seg = segment[:, ::8, ::8, :]
+        label_seg[label_seg>0] = 1
+        label_seg[label_seg<=0] = 0
+        minutiae_seg = (minutiae_o!=-1).astype(float)
+
+        # get ori & mnt
+        orientation = tra_ori_model.predict(alignment)
+        orientation = orientation/np.pi*180+90
+        orientation[orientation>=180.0] = 0.0 # orientation [0, 180)
+        minutiae_o = minutiae_o/np.pi*180+90 # [90, 450)
+        minutiae_o[minutiae_o>360] = minutiae_o[minutiae_o>360]-360 # to current coordinate system [0, 360)
+        minutiae_ori_o = np.copy(minutiae_o) # copy one
+        minutiae_ori_o[minutiae_ori_o>=180] = minutiae_ori_o[minutiae_ori_o>=180]-180 # for strong ori label [0,180)
+
+        # ori 2 gaussian
+        gaussian_pdf = signal.gaussian(361, 3)
+        y = np.reshape(np.arange(1, 180, 2), [1,1,1,-1])
+        delta = np.array(np.abs(orientation - y), dtype=int)
+        delta = np.minimum(delta, 180-delta)+180
+        label_ori = gaussian_pdf[delta]
+
+        # ori_o 2 gaussian
+        delta = np.array(np.abs(minutiae_ori_o - y), dtype=int)
+        delta = np.minimum(delta, 180-delta)+180
+        label_ori_o = gaussian_pdf[delta]
+
+        # mnt_o 2 gaussian
+        y = np.reshape(np.arange(1, 360, 2), [1,1,1,-1])
+        delta = np.array(np.abs(minutiae_o - y), dtype=int)
+        delta = np.minimum(delta, 360-delta)+180
+        label_mnt_o = gaussian_pdf[delta]
+
+        # w 2 gaussian
+        gaussian_pdf = signal.gaussian(17, 2)
+        y = np.reshape(np.arange(0, 8), [1,1,1,-1])
+        delta = (minutiae_w-y+8).astype(int)
+        label_mnt_w = gaussian_pdf[delta]
+
+        # h 2 gaussian
+        delta = (minutiae_h-y+8).astype(int)
+        label_mnt_h = gaussian_pdf[delta]
+
+        # mnt cls label -1:neg, 0:no care, 1:pos
+        label_mnt_s = np.copy(minutiae_seg)
+        label_mnt_s[label_mnt_s==0] = -1 # neg to -1
+        label_mnt_s = (label_mnt_s+ndimage.maximum_filter(label_mnt_s, size=(1,3,3,1)))/2 # around 3*3 pos -> 0
+
+        # apply segmentation
+        label_ori = label_ori * label_seg * have_alignment
+        label_ori_o = label_ori_o * minutiae_seg
+        label_mnt_o = label_mnt_o * minutiae_seg
+        label_mnt_w = label_mnt_w * minutiae_seg
+        label_mnt_h = label_mnt_h * minutiae_seg
+        yield image, label_ori, label_ori_o, label_seg, label_mnt_w, label_mnt_h, label_mnt_o, label_mnt_s, batch_name
+
+    if batch_size > 1 and use_multiprocessing==True:
+        p.close()
+        p.join()
+    return
+
+def merge_mul(x):
+    return reduce(lambda x,y:x*y, x)
+def merge_sum(x):
+    return reduce(lambda x,y:x+y, x)
+def reduce_sum(x):
+    return K.sum(x,axis=-1,keepdims=True)
+
+# Group with depth
+def merge_concat(x):
+    return K.tf.concat(x,3)
+def select_max(x):
+    x = x / (K.max(x, axis=-1, keepdims=True)+K.epsilon())
+    x = K.tf.where(K.tf.greater(x, 0.999), x, K.tf.zeros_like(x)) # select the biggest one
+    x = x / (K.sum(x, axis=-1, keepdims=True)+K.epsilon()) # prevent two or more ori is selected
+    return x
+
+
+kernal2angle = np.reshape(np.arange(1, 180, 2, dtype=float), [1,1,1,90])/90.*np.pi #2angle = angle*2
+sin2angle, cos2angle = np.sin(kernal2angle), np.cos(kernal2angle)
+def ori2angle(ori):
+    sin2angle_ori = K.sum(ori*sin2angle, -1, keepdims=True)
+    cos2angle_ori = K.sum(ori*cos2angle, -1, keepdims=True)
+    modulus_ori = K.sqrt(K.square(sin2angle_ori)+K.square(cos2angle_ori))
+    return sin2angle_ori, cos2angle_ori, modulus_ori
+
+
+# find highest peak using gaussian
+def ori_highest_peak(y_pred, length=180):
+    glabel = gausslabel(length=length,stride=2).astype(np.float32)
+    y_pred = tf.convert_to_tensor(y_pred, np.float32)
+    ori_gau = K.conv2d(y_pred,glabel,padding='same')
+    return ori_gau
+
+def ori_acc_delta_k(y_true, y_pred, k=10, max_delta=180):
+    # get ROI
+    label_seg = K.sum(y_true, axis=-1)
+    label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32)
+    # get pred angle
+    angle = K.cast(K.argmax(ori_highest_peak(y_pred, max_delta), axis=-1), dtype=K.tf.float32)*2.0+1.0
+    # get gt angle
+    angle_t = K.cast(K.argmax(y_true, axis=-1), dtype=K.tf.float32)*2.0+1.0
+    # get delta
+    angle_delta = K.abs(angle_t - angle)
+    acc = K.tf.less_equal(K.minimum(angle_delta, max_delta-angle_delta), k)
+    acc = K.cast(acc, dtype=K.tf.float32)
+    # apply ROI
+    acc = acc*label_seg
+    acc = K.sum(acc) / (K.sum(label_seg)+K.epsilon())
+    return acc
+def ori_acc_delta_10(y_true, y_pred):
+    return ori_acc_delta_k(y_true, y_pred, 10)
+def ori_acc_delta_20(y_true, y_pred):
+    return ori_acc_delta_k(y_true, y_pred, 20)
+def mnt_acc_delta_10(y_true, y_pred):
+    return ori_acc_delta_k(y_true, y_pred, 10, 360)
+def mnt_acc_delta_20(y_true, y_pred):
+    return ori_acc_delta_k(y_true, y_pred, 20, 360)
+
+def seg_acc_pos(y_true, y_pred):
+    y_true = K.tf.where(K.tf.less(y_true,0.0), K.tf.zeros_like(y_true), y_true)
+    acc = K.cast(K.equal(y_true, K.round(y_pred)), dtype=K.tf.float32)
+    acc = K.sum(acc * y_true) / (K.sum(y_true)+K.epsilon())
+    return acc
+def seg_acc_neg(y_true, y_pred):
+    y_true = K.tf.where(K.tf.less(y_true,0.0), K.tf.zeros_like(y_true), y_true)
+    acc = K.cast(K.equal(y_true, K.round(y_pred)), dtype=K.tf.float32)
+    acc = K.sum(acc * (1-y_true)) / (K.sum(1-y_true)+K.epsilon())
+    return acc
+def seg_acc_all(y_true, y_pred):
+    y_true = K.tf.where(K.tf.less(y_true,0.0), K.tf.zeros_like(y_true), y_true)
+    return K.mean(K.equal(y_true, K.round(y_pred)))
+
+def mnt_mean_delta(y_true, y_pred):
+    # get ROI
+    label_seg = K.sum(y_true, axis=-1)
+    label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32)
+    # get pred pos
+    pos = K.cast(K.argmax(y_pred, axis=-1), dtype=K.tf.float32)
+    # get gt pos
+    pos_t = K.cast(K.argmax(y_true, axis=-1), dtype=K.tf.float32)
+    # get delta
+    pos_delta = K.abs(pos_t - pos)
+    # apply ROI
+    pos_delta = pos_delta*label_seg
+    mean_delta = K.sum(pos_delta) / (K.sum(label_seg)+K.epsilon())
+    return mean_delta
+
+
+
+# currently can only produce one each time
+def label2mnt(mnt_s_out, mnt_w_out, mnt_h_out, mnt_o_out, thresh=0.5):
+    mnt_s_out = np.squeeze(mnt_s_out)
+    mnt_w_out = np.squeeze(mnt_w_out)
+    mnt_h_out = np.squeeze(mnt_h_out)
+    mnt_o_out = np.squeeze(mnt_o_out)
+    assert len(mnt_s_out.shape)==2 and len(mnt_w_out.shape)==3 and len(mnt_h_out.shape)==3 and len(mnt_o_out.shape)==3
+
+    # get cls results
+    mnt_sparse = sparse.coo_matrix(mnt_s_out>thresh)
+    mnt_list = np.array(zip(mnt_sparse.row, mnt_sparse.col), dtype=np.int32)
+    if mnt_list.shape[0] == 0:
+        return np.zeros((0, 4))
+
+    # get regression results
+    mnt_w_out = np.argmax(mnt_w_out, axis=-1)
+    mnt_h_out = np.argmax(mnt_h_out, axis=-1)
+    mnt_o_out = np.argmax(mnt_o_out, axis=-1) # TODO: use ori_highest_peak(np version)
+
+    # get final mnt
+    mnt_final = np.zeros((len(mnt_list), 4))
+    mnt_final[:, 0] = mnt_sparse.col*8 + mnt_w_out[mnt_list[:,0], mnt_list[:,1]]
+    mnt_final[:, 1] = mnt_sparse.row*8 + mnt_h_out[mnt_list[:,0], mnt_list[:,1]]
+    mnt_final[:, 2] = (mnt_o_out[mnt_list[:,0], mnt_list[:,1]]*2-89.)/180*np.pi
+    mnt_final[mnt_final[:, 2]<0.0, 2] = mnt_final[mnt_final[:, 2]<0.0, 2]+2*np.pi
+    # New one
+    mnt_final[:, 2] = (-mnt_final[:, 2]) % (2*np.pi)
+    mnt_final[:, 3] = mnt_s_out[mnt_list[:,0], mnt_list[:, 1]]
+
+    return mnt_final
+
+
+# image normalization
+def img_normalization(img_input, m0=0.0, var0=1.0):
+    m = K.mean(img_input, axis=[1,2,3], keepdims=True)
+    var = K.var(img_input, axis=[1,2,3], keepdims=True)
+    after = K.sqrt(var0*K.tf.square(img_input-m)/var)
+    image_n = K.tf.where(K.tf.greater(img_input, m), m0+after, m0-after)
+    return image_n
+
+# atan2 function
+def atan2(y_x):
+    y, x = y_x[0], y_x[1]+K.epsilon()
+    atan = K.tf.atan(y/x)
+    angle = K.tf.where(K.tf.greater(x,0.0), atan, K.tf.zeros_like(x))
+    angle = K.tf.where(K.tf.logical_and(K.tf.less(x,0.0),  K.tf.greater_equal(y,0.0)), atan+np.pi, angle)
+    angle = K.tf.where(K.tf.logical_and(K.tf.less(x,0.0),  K.tf.less(y,0.0)), atan-np.pi, angle)
+    return angle
+
+# traditional orientation estimation
+def orientation(image, stride=8, window=17):
+    with K.tf.name_scope('orientation'):
+        assert image.get_shape().as_list()[3] == 1, 'Images must be grayscale'
+        strides = [1, stride, stride, 1]
+        E = np.ones([window, window, 1, 1])
+        sobelx = np.reshape(np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], dtype=float), [3, 3, 1, 1])
+        sobely = np.reshape(np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]], dtype=float), [3, 3, 1, 1])
+        gaussian = np.reshape(gaussian2d((5, 5), 1), [5, 5, 1, 1])
+        with K.tf.name_scope('sobel_gradient'):
+            Ix = K.tf.nn.conv2d(image, sobelx, strides=[1,1,1,1], padding='SAME', name='sobel_x')
+            Iy = K.tf.nn.conv2d(image, sobely, strides=[1,1,1,1], padding='SAME', name='sobel_y')
+        with K.tf.name_scope('eltwise_1'):
+            Ix2 = K.tf.multiply(Ix, Ix, name='IxIx')
+            Iy2 = K.tf.multiply(Iy, Iy, name='IyIy')
+            Ixy = K.tf.multiply(Ix, Iy, name='IxIy')
+        with K.tf.name_scope('range_sum'):
+            Gxx = K.tf.nn.conv2d(Ix2, E, strides=strides, padding='SAME', name='Gxx_sum')
+            Gyy = K.tf.nn.conv2d(Iy2, E, strides=strides, padding='SAME', name='Gyy_sum')
+            Gxy = K.tf.nn.conv2d(Ixy, E, strides=strides, padding='SAME', name='Gxy_sum')
+        with K.tf.name_scope('eltwise_2'):
+            Gxx_Gyy = K.tf.subtract(Gxx, Gyy, name='Gxx_Gyy')
+            theta = atan2([2*Gxy, Gxx_Gyy]) + np.pi
+        # two-dimensional low-pass filter: Gaussian filter here
+        with K.tf.name_scope('gaussian_filter'):
+            phi_x = K.tf.nn.conv2d(K.tf.cos(theta), gaussian, strides=[1,1,1,1], padding='SAME', name='gaussian_x')
+            phi_y = K.tf.nn.conv2d(K.tf.sin(theta), gaussian, strides=[1,1,1,1], padding='SAME', name='gaussian_y')
+            theta = atan2([phi_y, phi_x])/2
+    return theta
+
+def get_tra_ori():
+    img_input=Input(shape=(None, None, 1))
+    theta = Lambda(orientation)(img_input)
+    model = Model(inputs=[img_input,], outputs=[theta,])
+    return model
+tra_ori_model = get_tra_ori()
+
+def get_maximum_img_size_and_names(dataset, sample_rate=None, max_size=None):
+
+    if isinstance(dataset, basestring):
+        dataset = [dataset]
+    if sample_rate is None:
+        sample_rate = [1]*len(dataset)
+    img_name, folder_name, img_size = [], [], []
+
+    for folder, rate in zip(dataset, sample_rate):
+        _, img_name_t = get_files_in_folder(folder, 'img_files/*'+'.bmp')
+        img_name.extend(img_name_t.tolist()*rate)
+        folder_name.extend([folder]*img_name_t.shape[0]*rate)
+
+        img_size.append(np.array(misc.imread(folder + 'img_files/' + img_name_t[0] + '.bmp', mode='L').shape))
+
+    img_name = np.asarray(img_name)
+    folder_name = np.asarray(folder_name)
+    img_size = np.max(np.asarray(img_size), axis=0)
+    # let img_size % 8 == 0
+    img_size = np.array(np.ceil(img_size / 8) * 8, dtype=np.int32)
+    return img_name, folder_name, img_size
+

minutiae/MinutiaeNet/CoarseNet/LossFunctions.py

@@ -0,0 +1,95 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+from keras.models import Model
+from keras.layers import Activation, AveragePooling2D, BatchNormalization, Concatenate, Conv2D, Dense, GlobalAveragePooling2D
+from keras.layers import Input, Lambda, MaxPooling2D
+from keras.applications.imagenet_utils import _obtain_input_shape
+from keras import backend as K
+import numpy as np
+from CoarseNet_utils import *
+
+def orientation_loss(y_true, y_pred, lamb=1.):
+    # clip
+    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
+    # get ROI
+    label_seg = K.sum(y_true, axis=-1, keepdims=True)
+    label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32)
+    # weighted cross entropy loss
+    lamb_pos, lamb_neg = 1., 1.
+    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
+    logloss = logloss*label_seg # apply ROI
+    logloss = -K.sum(logloss) / (K.sum(label_seg) + K.epsilon())
+
+    # coherence loss, nearby ori should be as near as possible
+    # Oritentation coherence loss
+
+    # 3x3 ones kernel
+    mean_kernal = np.reshape(np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32)/8, [3, 3, 1, 1])
+
+    sin2angle_ori, cos2angle_ori, modulus_ori = ori2angle(y_pred)
+    sin2angle = K.conv2d(sin2angle_ori, mean_kernal, padding='same')
+    cos2angle = K.conv2d(cos2angle_ori, mean_kernal, padding='same')
+    modulus = K.conv2d(modulus_ori, mean_kernal, padding='same')
+
+    coherence = K.sqrt(K.square(sin2angle) + K.square(cos2angle)) / (modulus + K.epsilon())
+    coherenceloss = K.sum(label_seg) / (K.sum(coherence*label_seg) + K.epsilon()) - 1
+    loss = logloss + lamb*coherenceloss
+    return loss
+
+def orientation_output_loss(y_true, y_pred):
+    # clip
+    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
+    # get ROI
+    label_seg = K.sum(y_true, axis=-1, keepdims=True)
+    label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32)
+    # weighted cross entropy loss
+    lamb_pos, lamb_neg= 1., 1.
+    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
+    logloss = logloss*label_seg # apply ROI
+    logloss = -K.sum(logloss) / (K.sum(label_seg) + K.epsilon())
+    return logloss
+
+def segmentation_loss(y_true, y_pred, lamb=1.):
+    # clip
+    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
+    # weighted cross entropy loss
+    total_elements = K.sum(K.tf.ones_like(y_true))
+    label_pos = K.tf.cast(K.tf.greater(y_true, 0.0), K.tf.float32)
+    lamb_pos = 0.5 * total_elements / K.sum(label_pos)
+    lamb_neg = 1 / (2 - 1/lamb_pos)
+    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
+    logloss = -K.mean(K.sum(logloss, axis=-1))
+    # smooth loss
+    smooth_kernal = np.reshape(np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]], dtype=np.float32)/8, [3, 3, 1, 1])
+    smoothloss = K.mean(K.abs(K.conv2d(y_pred, smooth_kernal)))
+    loss = logloss + lamb*smoothloss
+    return loss
+
+def minutiae_score_loss(y_true, y_pred):
+    # clip
+    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
+    # get ROI
+    label_seg = K.tf.cast(K.tf.not_equal(y_true, 0.0), K.tf.float32)
+    y_true = K.tf.where(K.tf.less(y_true,0.0), K.tf.zeros_like(y_true), y_true) # set -1 -> 0
+    # weighted cross entropy loss
+    total_elements = K.sum(label_seg) + K.epsilon()
+    lamb_pos, lamb_neg = 10., .5
+    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
+    # apply ROI
+    logloss = logloss*label_seg
+    logloss = -K.sum(logloss) / total_elements
+    return logloss

minutiae/MinutiaeNet/CoarseNet/MinutiaeNet_utils.py

@@ -0,0 +1,810 @@
+import os
+
+import glob
+import shutil
+import logging
+import matplotlib.pyplot as plt
+import numpy as np
+from scipy import ndimage, misc, signal, spatial
+from skimage.filters import gaussian
+import cv2
+import math
+
+def mkdir(path):
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+def re_mkdir(path):
+    if os.path.exists(path):
+        shutil.rmtree(path)
+    os.makedirs(path)
+
+def init_log(output_dir):
+    re_mkdir(output_dir)
+    logging.basicConfig(level=logging.DEBUG,
+        format='%(asctime)s %(message)s',
+        datefmt='%Y%m%d-%H:%M:%S',
+        filename=os.path.join(output_dir, 'log.log'),
+        filemode='w')
+    console = logging.StreamHandler()
+    console.setLevel(logging.INFO)
+    logging.getLogger('').addHandler(console)
+    return logging
+
+def copy_file(path_s, path_t):
+    shutil.copy(path_s, path_t)   
+
+def get_files_in_folder(folder, file_ext=None):
+    files = glob.glob(os.path.join(folder, "*" + file_ext))
+    files_name = []
+    for i in files:
+        _, name = os.path.split(i)
+        name, ext = os.path.splitext(name)
+        files_name.append(name)
+    return np.asarray(files), np.asarray(files_name)
+
+def point_rot(points, theta, b_size, a_size):
+    cosA = np.cos(theta)
+    sinA = np.sin(theta)
+    b_center = [b_size[1]/2.0, b_size[0]/2.0]
+    a_center = [a_size[1]/2.0, a_size[0]/2.0]
+    points = np.dot(points-b_center, np.array([[cosA,-sinA],[sinA,cosA]]))+a_center
+    return points
+
+def mnt_reader(file_name):
+    f = open(file_name)
+    minutiae = []
+    for i, line in enumerate(f):
+        if i < 4 or len(line) == 0: continue
+        w, h, o = [float(x) for x in line.split()]
+        w, h = int(round(w)), int(round(h))
+        minutiae.append([w, h, o])
+    f.close()
+    return minutiae
+
+def mnt_writer(mnt, image_name, image_size, file_name):
+    f = open(file_name, 'w')
+    f.write('%s\n'%(image_name))
+    f.write('%d %d %d\n'%(mnt.shape[0], image_size[0], image_size[1]))
+    for i in xrange(mnt.shape[0]):
+        f.write('%d %d %.6f %.4f\n'%(mnt[i,0], mnt[i,1], mnt[i,2], mnt[i,3]))
+    f.close()
+    return
+
+def gabor_fn(ksize, sigma, theta, Lambda, psi, gamma):
+    sigma_x = sigma
+    sigma_y = float(sigma) / gamma
+    # Bounding box
+    nstds = 3
+    xmax = ksize[0]/2
+    ymax = ksize[1]/2
+    xmin = -xmax
+    ymin = -ymax
+    (y, x) = np.meshgrid(np.arange(ymin, ymax + 1), np.arange(xmin, xmax + 1))
+    # Rotation
+    x_theta = x * np.cos(theta) + y * np.sin(theta)
+    y_theta = -x * np.sin(theta) + y * np.cos(theta)
+    gb_cos = np.exp(-.5 * (x_theta ** 2 / sigma_x ** 2 + y_theta ** 2 / sigma_y ** 2)) * np.cos(2 * np.pi / Lambda * x_theta + psi)
+    gb_sin = np.exp(-.5 * (x_theta ** 2 / sigma_x ** 2 + y_theta ** 2 / sigma_y ** 2)) * np.sin(2 * np.pi / Lambda * x_theta + psi)
+    return gb_cos, gb_sin
+
+def gabor_bank(stride=2,Lambda=8):
+
+    filters_cos = np.ones([25,25,180/stride], dtype=float)
+    filters_sin = np.ones([25,25,180/stride], dtype=float)
+
+    for n, i in enumerate(xrange(-90,90,stride)):
+        theta = i*np.pi/180.
+        kernel_cos, kernel_sin = gabor_fn((24,24),4.5, -theta, Lambda, 0, 0.5)
+        filters_cos[..., n] = kernel_cos
+        filters_sin[..., n] = kernel_sin
+
+    filters_cos = np.reshape(filters_cos,[25,25,1,-1])
+    filters_sin = np.reshape(filters_sin,[25,25,1,-1])
+    return filters_cos, filters_sin
+
+def gaussian2d(shape=(5,5),sigma=0.5):
+    """
+    2D gaussian mask - should give the same result as MATLAB's
+    fspecial('gaussian',[shape],[sigma])
+    """
+    m,n = [(ss-1.)/2. for ss in shape]
+    y,x = np.ogrid[-m:m+1,-n:n+1]
+    h = np.exp( -(x*x + y*y) / (2.*sigma*sigma) )
+    h[ h < np.finfo(h.dtype).eps*h.max() ] = 0
+    sumh = h.sum()
+    if sumh != 0:
+        h /= sumh
+    return h
+
+def gausslabel(length=180, stride=2):
+    gaussian_pdf = signal.gaussian(length+1, 3)
+    label = np.reshape(np.arange(stride/2, length, stride), [1,1,-1,1])
+    y = np.reshape(np.arange(stride/2, length, stride), [1,1,1,-1])
+    delta = np.array(np.abs(label - y), dtype=int)
+    delta = np.minimum(delta, length-delta)+length/2
+    return gaussian_pdf[delta]
+
+def angle_delta(A, B, max_D=np.pi*2):
+    delta = np.abs(A - B)
+    delta = np.minimum(delta, max_D-delta)
+    return delta
+def fmeasure(P, R):
+    return 2*P*R/(P+R+1e-10)
+def distance(y_true, y_pred, max_D=16, max_O=np.pi/6):
+    D = spatial.distance.cdist(y_true[:, :2], y_pred[:, :2], 'euclidean')
+    O = spatial.distance.cdist(np.reshape(y_true[:, 2], [-1, 1]), np.reshape(y_pred[:, 2], [-1, 1]), angle_delta)
+    return (D<=max_D)*(O<=max_O)
+
+def metric_P_R_F(y_true, y_pred, maxd=16, maxo=np.pi/6):
+    # Calculate Precision, Recall, F-score
+    if y_pred.shape[0]==0 or y_true.shape[0]==0:
+        return 0,0,0,0,0
+
+    y_true, y_pred = np.array(y_true), np.array(y_pred)
+    total_gt, total = float(y_true.shape[0]), float(y_pred.shape[0])
+    # Using L2 loss
+    dis = spatial.distance.cdist(y_pred[:, :2], y_true[:, :2], 'euclidean')
+    mindis,idx = dis.min(axis=1),dis.argmin(axis=1)
+
+    #Change to adapt to new annotation: old version. When training, comment it
+    # y_pred[:,2] = -y_pred[:,2]
+
+    angle = abs(np.mod(y_pred[:,2],2*np.pi) - y_true[idx,2])
+    angle = np.asarray([angle, 2*np.pi-angle]).min(axis=0)
+
+    # Satisfy the threshold
+    tmp=(mindis <= maxd) & (angle<=maxo)
+    #print('mindis,idx,angle,tmp=%s,%s,%s,%s'%(mindis,idx,angle,tmp))
+
+    precision = len(np.unique(idx[(mindis <= maxd) & (angle<=maxo)]))/float(y_pred.shape[0])
+    recall = len(np.unique(idx[(mindis <= maxd) & (angle<=maxo)]))/float(y_true.shape[0])
+    #print('pre=%f/ %f'%(len(np.unique(idx[(mindis <= maxd) & (angle<=maxo)])),float(y_pred.shape[0])))
+    #print('recall=%f/ %f'%(len(np.unique(idx[(mindis <= maxd) & (angle<=maxo)])),float(y_true.shape[0])))
+    if recall!=0:
+        loc = np.mean(mindis[(mindis <= maxd) & (angle<=maxo)])
+        ori = np.mean(angle[(mindis <= maxd) & (angle<=maxo)])
+    else:
+        loc = 0
+        ori = 0
+    return precision, recall, fmeasure(precision, recall), loc, ori
+
+def nms(mnt):
+    if mnt.shape[0]==0:
+        return mnt
+    # sort score
+    mnt_sort = mnt.tolist()
+    mnt_sort.sort(key=lambda x:x[3], reverse=True)
+    mnt_sort = np.array(mnt_sort)
+    # cal distance
+    inrange = distance(mnt_sort, mnt_sort, max_D=16, max_O=np.pi/6).astype(np.float32)
+    keep_list = np.ones(mnt_sort.shape[0])
+    for i in xrange(mnt_sort.shape[0]):
+        if keep_list[i] == 0:
+            continue
+        keep_list[i+1:] = keep_list[i+1:]*(1-inrange[i, i+1:])
+    return mnt_sort[keep_list.astype(np.bool), :]
+
+def fuse_nms(mnt, mnt_set_2):
+    if mnt.shape[0]==0:
+        return mnt
+    # sort score
+    all_mnt = np.concatenate((mnt, mnt_set_2))
+
+    mnt_sort = all_mnt.tolist()
+    mnt_sort.sort(key=lambda x:x[3], reverse=True)
+    mnt_sort = np.array(mnt_sort)
+    # cal distance
+    inrange = distance(mnt_sort, mnt_sort, max_D=16, max_O=2*np.pi).astype(np.float32)
+    keep_list = np.ones(mnt_sort.shape[0])
+    for i in xrange(mnt_sort.shape[0]):
+        if keep_list[i] == 0:
+            continue
+        keep_list[i+1:] = keep_list[i+1:]*(1-inrange[i, i+1:])
+    return mnt_sort[keep_list.astype(np.bool), :]
+
+
+def py_cpu_nms(det, thresh):
+    if det.shape[0]==0:
+        return det
+    dets = det.tolist()
+    dets.sort(key=lambda x:x[3], reverse=True)
+    dets = np.array(dets)
+
+    box_sz = 25
+    x1 = np.reshape(dets[:,0],[-1,1]) -box_sz
+    y1 = np.reshape(dets[:,1],[-1,1]) -box_sz
+    x2 = np.reshape(dets[:,0],[-1,1]) +box_sz
+    y2 = np.reshape(dets[:,1],[-1,1]) +box_sz
+    scores = dets[:, 2]
+
+    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
+    order = scores.argsort()[::-1]
+
+    keep = []
+    while order.size > 0:
+        i = order[0]
+        keep.append(i)
+        xx1 = np.maximum(x1[i], x1[order[1:]])
+        yy1 = np.maximum(y1[i], y1[order[1:]])
+        xx2 = np.minimum(x2[i], x2[order[1:]])
+        yy2 = np.minimum(y2[i], y2[order[1:]])
+
+        w = np.maximum(0.0, xx2 - xx1 + 1)
+        h = np.maximum(0.0, yy2 - yy1 + 1)
+        inter = w * h
+        ovr = inter / (areas[i] + areas[order[1:]] - inter)
+
+        inds = np.where(ovr <= thresh)[0]
+        order = order[inds + 1]
+
+    return dets[keep, :]
+
+def draw_minutiae(image, minutiae, fname, saveimage= False, r=15, drawScore=False):
+    image = np.squeeze(image)
+    fig = plt.figure()
+    
+
+    plt.imshow(image,cmap='gray')
+    plt.hold(True)
+    # Check if no minutiae
+    if minutiae.shape[0] > 0:
+        plt.plot(minutiae[:, 0], minutiae[:, 1], 'rs', fillstyle='none', linewidth=1)
+        for x, y, o, s in minutiae:
+            plt.plot([x, x+r*np.cos(o)], [y, y+r*np.sin(o)], 'r-')
+            if drawScore == True:
+                plt.text(x - 10, y - 10, '%.2f' % s, color='yellow', fontsize=4)
+
+    plt.axis([0,image.shape[1],image.shape[0],0])
+    plt.axis('off')
+    if saveimage:
+        plt.savefig(fname, dpi=500, bbox_inches='tight', pad_inches = 0)
+        plt.close(fig)
+    else:
+        plt.show()
+    return
+
+def draw_minutiae_overlay(image, minutiae, mnt_gt, fname, saveimage= False, r=15, drawScore=False):
+    image = np.squeeze(image)
+    fig = plt.figure()
+    
+
+    plt.imshow(image,cmap='gray')
+    plt.hold(True)
+
+    if mnt_gt.shape[1] > 3:
+        mnt_gt = mnt_gt[:,:3]
+
+    if mnt_gt.shape[0] > 0:
+        if mnt_gt.shape[1] > 3:
+            mnt_gt = mnt_gt[:, :3]
+        plt.plot(mnt_gt[:, 0], mnt_gt[:, 1], 'bs', fillstyle='none', linewidth=1)
+        for x, y, o in mnt_gt:
+            plt.plot([x, x+r*np.cos(o)], [y, y+r*np.sin(o)], 'b-')
+
+    if minutiae.shape[0] > 0:
+        plt.plot(minutiae[:, 0], minutiae[:, 1], 'rs', fillstyle='none', linewidth=1)
+        for x, y, o in minutiae:
+            plt.plot([x, x+r*np.cos(o)], [y, y+r*np.sin(o)], 'r-')
+            if drawScore == True:
+                plt.text(x - 10, y - 10, '%.2f' % s, color='yellow', fontsize=4)
+
+    plt.axis([0,image.shape[1],image.shape[0],0])
+    plt.axis('off')
+    plt.show()
+    if saveimage:
+        plt.savefig(fname, dpi=500, bbox_inches='tight')
+        plt.close(fig)
+    else:
+        plt.show()
+    return
+
+
+def draw_minutiae_overlay_with_score(image, minutiae, mnt_gt, fname, saveimage=False, r=15):
+    image = np.squeeze(image)
+    fig = plt.figure()
+
+    plt.imshow(image, cmap='gray')
+    plt.hold(True)
+
+
+    if mnt_gt.shape[0] > 0:
+        plt.plot(mnt_gt[:, 0], mnt_gt[:, 1], 'bs', fillstyle='none', linewidth=1)
+        if mnt_gt.shape[1] > 3:
+            for x, y, o, s in mnt_gt:
+                plt.plot([x, x + r * np.cos(o)], [y, y + r * np.sin(o)], 'b-')
+                plt.text(x - 10, y - 5, '%.2f' % s, color='green', fontsize=4)
+        else:
+            for x, y, o in mnt_gt:
+                plt.plot([x, x + r * np.cos(o)], [y, y + r * np.sin(o)], 'b-')
+
+    if minutiae.shape[0] > 0:
+        plt.plot(minutiae[:, 0], minutiae[:, 1], 'rs', fillstyle='none', linewidth=1)
+        for x, y, o, s in minutiae:
+            plt.plot([x, x + r * np.cos(o)], [y, y + r * np.sin(o)], 'r-')
+            plt.text(x-10,y-10,'%.2f'%s,color='yellow',fontsize=4)
+
+    plt.axis([0, image.shape[1], image.shape[0], 0])
+    plt.axis('off')
+
+    if saveimage:
+        plt.savefig(fname, dpi=500, bbox_inches='tight')
+        plt.close(fig)
+    else:
+        plt.show()
+    return
+
+def draw_ori_on_img(img, ori, mask, fname, saveimage=False, coh=None, stride=16):
+    ori = np.squeeze(ori)
+    #mask = np.squeeze(np.round(mask))
+
+    img = np.squeeze(img)
+    ori = ndimage.zoom(ori, np.array(img.shape)/np.array(ori.shape, dtype=float), order=0)
+    if mask.shape != img.shape:
+        mask = ndimage.zoom(mask, np.array(img.shape)/np.array(mask.shape, dtype=float), order=0)
+    if coh is None:
+        coh = np.ones_like(img)
+    fig = plt.figure()
+    plt.imshow(img,cmap='gray')
+    plt.hold(True)  
+    for i in xrange(stride,img.shape[0],stride):
+        for j in xrange(stride,img.shape[1],stride):
+            if mask[i, j] == 0:
+                continue
+            x, y, o, r = j, i, ori[i,j], coh[i,j]*(stride*0.9)
+            plt.plot([x, x+r*np.cos(o)], [y, y+r*np.sin(o)], 'r-')
+    plt.axis([0,img.shape[1],img.shape[0],0])
+    plt.axis('off')
+    if saveimage:
+        plt.savefig(fname, bbox_inches='tight')
+        plt.close(fig)
+    else:
+        plt.show()
+    return
+
+def local_constrast_enhancement(img):
+    img = img.astype(np.float32)
+    meanV = cv2.blur(img,(15,15))
+    normalized = img - meanV
+    var = abs(normalized)
+
+    var = cv2.blur(var,(15,15))
+    normalized = normalized/(var+10) *0.75
+    normalized = np.clip(normalized, -1, 1)
+    normalized = (normalized+1)*127.5
+    return normalized
+
+def get_quality_map_ori_dict(img, dict, spacing, dir_map = None, block_size = 16):
+    if img.dtype=='uint8':
+        img = img.astype(np.float)
+    img = FastEnhanceTexture(img)
+    h, w = img.shape
+    blkH, blkW = dir_map.shape
+
+    quality_map = np.zeros((blkH,blkW),dtype=np.float)
+    fre_map = np.zeros((blkH,blkW),dtype=np.float)
+    ori_num = len(dict)
+    #dir_map = math.pi/2 - dir_map
+    dir_ind = dir_map*ori_num/math.pi
+    dir_ind = dir_ind.astype(np.int)
+    dir_ind = dir_ind%ori_num
+
+    patch_size = np.sqrt(dict[0].shape[1])
+    patch_size = patch_size.astype(np.int)
+    pad_size = (patch_size-block_size)//2
+    img = np.lib.pad(img, (pad_size, pad_size), 'symmetric')
+    for i in range(0,blkH):
+        for j in range(0,blkW):
+            ind = dir_ind[i,j]
+            patch = img[i*block_size:i*block_size+patch_size,j*block_size:j*block_size+patch_size]
+
+            patch = patch.reshape(patch_size*patch_size,)
+            patch = patch - np.mean(patch)
+            patch = patch / (np.linalg.norm(patch)+0.0001)
+            patch[patch>0.05] = 0.05
+            patch[patch<-0.05] = -0.05
+
+            simi = np.dot(dict[ind], patch)
+            similar_ind = np.argmax(abs(simi))
+            quality_map[i,j] = np.max(abs(simi))
+            fre_map[i,j] = 1./spacing[ind][similar_ind]
+
+    quality_map = gaussian(quality_map,sigma=2)
+    return quality_map, fre_map
+
+def FastEnhanceTexture(img,sigma=2.5,show=False):
+    img = img.astype(np.float32)
+    h, w = img.shape
+    h2 = 2 ** nextpow2(h)
+    w2 = 2 ** nextpow2(w)
+
+    FFTsize = np.max([h2, w2])
+    x, y = np.meshgrid(range(-FFTsize / 2, FFTsize / 2), range(-FFTsize / 2, FFTsize / 2))
+    r = np.sqrt(x * x + y * y) + 0.0001
+    r = r/FFTsize
+
+    L = 1. / (1 + (2 * math.pi * r * sigma)** 4)
+    img_low = LowpassFiltering(img, L)
+
+    gradim1=  compute_gradient_norm(img)
+    gradim1 = LowpassFiltering(gradim1,L)
+
+    gradim2=  compute_gradient_norm(img_low)
+    gradim2 = LowpassFiltering(gradim2,L)
+
+    diff = gradim1-gradim2
+    ar1 = np.abs(gradim1)
+    diff[ar1>1] = diff[ar1>1]/ar1[ar1>1]
+    diff[ar1 <= 1] = 0
+
+    cmin = 0.3
+    cmax = 0.7
+
+    weight = (diff-cmin)/(cmax-cmin)
+    weight[diff<cmin] = 0
+    weight[diff>cmax] = 1
+
+
+    u = weight * img_low + (1-weight)* img
+
+    temp = img - u
+
+    lim = 20
+
+    temp1 = (temp + lim) * 255 / (2 * lim)
+
+    temp1[temp1 < 0] = 0
+    temp1[temp1 >255] = 255
+    v = temp1
+    if show:
+        plt.imshow(v,cmap='gray')
+        plt.show()
+    return v
+
+def compute_gradient_norm(input):
+    input = input.astype(np.float32)
+
+    Gx, Gy = np.gradient(input)
+    out = np.sqrt(Gx * Gx + Gy * Gy) + 0.000001
+    return out
+
+def LowpassFiltering(img,L):
+    h,w = img.shape
+    h2,w2 = L.shape
+
+    img = cv2.copyMakeBorder(img, 0, h2-h, 0, w2-w, cv2.BORDER_CONSTANT, value=0)
+
+    img_fft = np.fft.fft2(img)
+    img_fft = np.fft.fftshift(img_fft)
+
+    img_fft = img_fft * L
+    rec_img = np.fft.ifft2(np.fft.fftshift(img_fft))
+    rec_img = np.real(rec_img)
+    rec_img = rec_img[:h,:w]
+
+    return rec_img
+
+def nextpow2(x):
+    return int(math.ceil(math.log(x, 2)))
+
+def construct_dictionary(ori_num = 30):
+    ori_dict = []
+    s = []
+    for i in range(ori_num):
+        ori_dict.append([])
+        s.append([])
+
+    patch_size2 = 16
+    patch_size = 32
+    dict_all = []
+    spacing_all = []
+    ori_all = []
+    Y, X = np.meshgrid(range(-patch_size2,patch_size2), range(-patch_size2,patch_size2))
+
+    for spacing in range(6,13):
+        for valley_spacing in range(3,spacing//2):
+            ridge_spacing = spacing - valley_spacing
+            for k in range(ori_num):
+                theta = np.pi/2-k*np.pi / ori_num
+                X_r = X * np.cos(theta) - Y * np.sin(theta)
+                for offset in range(0,spacing-1,2):
+                    X_r_offset = X_r + offset + ridge_spacing / 2
+                    X_r_offset = np.remainder(X_r_offset, spacing)
+                    Y1 = np.zeros((patch_size, patch_size))
+                    Y2 = np.zeros((patch_size, patch_size))
+                    Y1[X_r_offset <= ridge_spacing] = X_r_offset[X_r_offset <= ridge_spacing]
+                    Y2[X_r_offset > ridge_spacing] = X_r_offset[X_r_offset > ridge_spacing] - ridge_spacing
+                    element = -np.sin(2 * math.pi * (Y1 / ridge_spacing / 2)) + np.sin(2 * math.pi * (Y2 / valley_spacing / 2))
+
+                    element = element.reshape(patch_size*patch_size,)
+                    element = element-np.mean(element)
+                    element  = element/ np.linalg.norm(element)
+                    ori_dict[k].append(element)
+                    s[k].append(spacing)
+                    dict_all.append(element)
+                    spacing_all.append(1.0/spacing)
+                    ori_all.append(theta)
+
+    for i in range(len(ori_dict)):
+        ori_dict[i] = np.asarray(ori_dict[i])
+        s[k] = np.asarray(s[k])
+
+    dict_all = np.asarray(dict_all)
+    dict_all = np.transpose(dict_all)
+    spacing_all = np.asarray(spacing_all)
+    ori_all = np.asarray(ori_all)
+
+
+    return ori_dict, s, dict_all, ori_all,spacing_all
+
+def get_maps_STFT(img,patch_size = 64,block_size = 16, preprocess = False):
+    assert len(img.shape) == 2
+
+    nrof_dirs = 16
+    ovp_size = (patch_size-block_size)//2
+    if preprocess:
+        img = FastEnhanceTexture(img, sigma=2.5, show=False)
+
+    img = np.lib.pad(img, (ovp_size,ovp_size),'symmetric')
+    h,w = img.shape
+    blkH = (h - patch_size)//block_size+1
+    blkW = (w - patch_size)//block_size+1
+    local_info = np.empty((blkH,blkW),dtype = object)
+
+    x, y = np.meshgrid(range(-patch_size / 2,patch_size / 2), range(-patch_size / 2,patch_size / 2))
+    x = x.astype(np.float32)
+    y = y.astype(np.float32)
+    r = np.sqrt(x*x + y*y) + 0.0001
+
+
+    RMIN = 3  # min allowable ridge spacing
+    RMAX = 18 # maximum allowable ridge spacing
+    FLOW = patch_size / RMAX
+    FHIGH = patch_size / RMIN
+    dRLow = 1. / (1 + (r / FHIGH) ** 4)
+    dRHigh = 1. / (1 + (FLOW / r) ** 4)
+    dBPass = dRLow * dRHigh  # bandpass
+
+    dir = np.arctan2(y,x)
+    dir[dir<0] = dir[dir<0] + math.pi
+    dir_ind = np.floor(dir/(math.pi/nrof_dirs))
+    dir_ind = dir_ind.astype(np.int,copy=False)
+    dir_ind[dir_ind==nrof_dirs] = 0
+
+
+    dir_ind_list = []
+    for i in range(nrof_dirs):
+        tmp = np.argwhere(dir_ind == i)
+        dir_ind_list.append(tmp)
+
+
+    sigma = patch_size/3
+    weight = np.exp(-(x*x + y*y)/(sigma*sigma))
+
+
+    for i in range(0,blkH):
+        for j in range(0,blkW):
+            patch =img[i*block_size:i*block_size+patch_size,j*block_size:j*block_size+patch_size].copy()
+            local_info[i,j] = local_STFT(patch,weight,dBPass)
+            local_info[i, j].analysis(r,dir_ind_list)
+
+
+    # get the ridge flow from the local information
+    dir_map,fre_map = get_ridge_flow_top(local_info)
+    dir_map = smooth_dir_map(dir_map)
+
+    return dir_map, fre_map
+
+def smooth_dir_map(dir_map,sigma=2.0,mask = None):
+
+    cos2Theta = np.cos(dir_map * 2)
+    sin2Theta = np.sin(dir_map * 2)
+    if mask is not None:
+        assert (dir_map.shape[0] == mask.shape[0])
+        assert (dir_map.shape[1] == mask.shape[1])
+        cos2Theta[mask == 0] = 0
+        sin2Theta[mask == 0] = 0
+
+    cos2Theta = gaussian(cos2Theta, sigma, multichannel=False, mode='reflect')
+    sin2Theta = gaussian(sin2Theta, sigma, multichannel=False, mode='reflect')
+
+    dir_map = np.arctan2(sin2Theta,cos2Theta)*0.5
+
+
+    return dir_map
+
+def get_ridge_flow_top(local_info):
+
+    blkH,blkW = local_info.shape
+    dir_map = np.zeros((blkH,blkW)) - 10
+    fre_map = np.zeros((blkH, blkW)) - 10
+    for i in range(blkH):
+        for j in range(blkW):
+            if local_info[i,j].ori is None:
+                continue
+
+            dir_map[i,j] = local_info[i,j].ori[0] #+ math.pi*0.5
+            fre_map[i,j] = local_info[i,j].fre[0]
+    return dir_map,fre_map
+
+
+class local_STFT:
+    def __init__(self,patch,weight = None, dBPass = None):
+
+
+        if weight is not None:
+            patch = patch * weight
+        patch = patch - np.mean(patch)
+        norm = np.linalg.norm(patch)
+        patch = patch / (norm+0.000001)
+
+        f = np.fft.fft2(patch)
+        fshift = np.fft.fftshift(f)
+        if dBPass is not None:
+            fshift = dBPass * fshift
+
+        self.patch_FFT = fshift
+        self.patch = patch
+        self.ori = None
+        self.fre = None
+        self.confidence = None
+        self.patch_size = patch.shape[0]
+
+    def analysis(self,r,dir_ind_list=None,N=2):
+
+        assert(dir_ind_list is not None)
+        energy = np.abs(self.patch_FFT)
+        energy = energy / (np.sum(energy)+0.00001)
+        nrof_dirs = len(dir_ind_list)
+
+        ori_interval = math.pi/nrof_dirs
+        ori_interval2 = ori_interval/2
+
+
+        pad_size = 1
+        dir_norm = np.zeros((nrof_dirs + 2,))
+        for i in range(nrof_dirs):
+            tmp = energy[dir_ind_list[i][:, 0], dir_ind_list[i][:, 1]]
+            dir_norm[i + 1] = np.sum(tmp)
+
+        dir_norm[0] = dir_norm[nrof_dirs]
+        dir_norm[nrof_dirs + 1] = dir_norm[1]
+
+        # smooth dir_norm
+        smoothed_dir_norm = dir_norm
+        for i in range(1, nrof_dirs + 1):
+            smoothed_dir_norm[i] = (dir_norm[i - 1] + dir_norm[i] * 4 + dir_norm[i + 1]) / 6
+
+        smoothed_dir_norm[0] = smoothed_dir_norm[nrof_dirs]
+        smoothed_dir_norm[nrof_dirs + 1] = smoothed_dir_norm[1]
+
+        den = np.sum(smoothed_dir_norm[1:nrof_dirs + 1]) + 0.00001  # verify if den == 1
+        smoothed_dir_norm = smoothed_dir_norm/den  # normalization if den == 1, this line can be removed
+
+        ori = []
+        fre = []
+        confidence = []
+
+        wenergy = energy*r
+        for i in range(1, nrof_dirs+1):
+            if smoothed_dir_norm[i] > smoothed_dir_norm[i-1] and smoothed_dir_norm[i] > smoothed_dir_norm[i+1]:
+                tmp_ori = (i-pad_size)*ori_interval + ori_interval2 + math.pi/2
+                ori.append(tmp_ori)
+                confidence.append(smoothed_dir_norm[i])
+                tmp_fre = np.sum(wenergy[dir_ind_list[i-pad_size][:, 0], dir_ind_list[i-pad_size][:, 1]])/dir_norm[i]
+                tmp_fre = 1/(tmp_fre+0.00001)
+                fre.append(tmp_fre)
+
+
+        if len(confidence)>0:
+            confidence = np.asarray(confidence)
+            fre = np.asarray(fre)
+            ori = np.asarray(ori)
+            ind = confidence.argsort()[::-1]
+            confidence = confidence[ind]
+            fre = fre[ind]
+            ori = ori[ind]
+            if len(confidence) >= 2 and confidence[0]/confidence[1]>2.0:
+
+                self.ori = [ori[0]]
+                self.fre = [fre[0]]
+                self.confidence = [confidence[0]]
+            elif len(confidence)>N:
+                fre = fre[:N]
+                ori = ori[:N]
+                confidence = confidence[:N]
+                self.ori = ori
+                self.fre = fre
+                self.confidence = confidence
+            else:
+                self.ori = ori
+                self.fre = fre
+                self.confidence = confidence
+
+    def get_features_of_topN(self,N=2):
+        if self.confidence is None:
+            self.border_wave = None
+            return
+        candi_num = len(self.ori)
+        candi_num = np.min([candi_num,N])
+        patch_size = self.patch_FFT.shape
+        for i in range(candi_num):
+
+            kernel = gabor_kernel(self.fre[i], theta=self.ori[i], sigma_x=10, sigma_y=10)
+
+            kernel_f = np.fft.fft2(kernel.real, patch_size)
+            kernel_f = np.fft.fftshift(kernel_f)
+            patch_f = self.patch_FFT * kernel_f
+
+            patch_f = np.fft.ifftshift(patch_f)  # *np.sqrt(np.abs(fshift)))
+            rec_patch = np.real(np.fft.ifft2(patch_f))
+
+
+            plt.subplot(121), plt.imshow(self.patch, cmap='gray')
+            plt.title('Input patch'), plt.xticks([]), plt.yticks([])
+            plt.subplot(122), plt.imshow(rec_patch, cmap='gray')
+            plt.title('filtered patch'), plt.xticks([]), plt.yticks([])
+            plt.show()
+
+    def reconstruction(self,weight=None):
+        f_ifft = np.fft.ifftshift(self.patch_FFT)  # *np.sqrt(np.abs(fshift)))
+        rec_patch = np.real(np.fft.ifft2(f_ifft))
+        if weight is not None:
+            rec_patch = rec_patch * weight
+        return rec_patch
+
+    def gabor_filtering(self,theta,fre,weight=None):
+
+        patch_size = self.patch_FFT.shape
+        kernel = gabor_kernel(fre, theta=theta,sigma_x=4,sigma_y=4)
+
+        f = kernel.real
+        f = f - np.mean(f)
+        f = f / (np.linalg.norm(f)+0.0001)
+
+
+        kernel_f = np.fft.fft2(f,patch_size)
+        kernel_f = np.fft.fftshift(kernel_f)
+        patch_f = self.patch_FFT*kernel_f
+
+        patch_f = np.fft.ifftshift(patch_f)  # *np.sqrt(np.abs(fshift)))
+        rec_patch = np.real(np.fft.ifft2(patch_f))
+        if weight is not None:
+            rec_patch = rec_patch * weight
+        return rec_patch
+
+
+def show_orientation_field(img,dir_map,mask=None,fname=None):
+    h,w = img.shape[:2]
+
+    if mask is None:
+        mask = np.ones((h,w),dtype=np.uint8)
+    blkH, blkW = dir_map.shape
+
+    blk_size = h/blkH
+
+    R = blk_size/2*0.8
+    fig, ax = plt.subplots(1)
+    ax.imshow(img, cmap='gray')
+    for i in range(blkH):
+        y0 = i*blk_size + blk_size/2
+        y0 = int(y0)
+        for j in range(blkW):
+            x0 = j*blk_size + blk_size/2
+            x0 = int(x0)
+            ori = dir_map[i,j]
+            if mask[y0,x0] == 0:
+                continue
+            if ori<-9:
+                continue
+            x1 = x0 - R * math.cos(ori)
+            x2 = x0 + R * math.cos(ori)
+            y1 = y0 - R * math.sin(ori)
+            y2 = y0 + R * math.sin(ori)
+            plt.plot([x1, x2], [y1, y2], 'r-', lw=2)
+    plt.axis('off')
+    if fname is not None:
+        fig.savefig(fname,dpi = 500, bbox_inches='tight', pad_inches = 0)
+        plt.close()
+    else:
+        plt.show(block=True)

minutiae/MinutiaeNet/Dataset/CoarseNet_test/mnt_files/1.mnt

@@ -0,0 +1,18 @@
+001
+16 800 768
+313 382 3.141593e-01
+353 385 3.665191e-01
+261 384 3.141593e-01
+287 327 4.188790e-01
+356 353 4.014257e-01
+385 197 6.632251e-01
+418 85 6.632251e-01
+397 307 7.679449e-01
+346 302 7.155850e-01
+418 267 7.679449e-01
+418 235 7.155850e-01
+473 233 8.552113e-01
+458 182 3.822271e+00
+349 277 3.787364e+00
+418 277 3.892084e+00
+453 235 3.839724e+00

minutiae/MinutiaeNet/Dataset/CoarseNet_test/mnt_files/2.mnt

@@ -0,0 +1,18 @@
+002
+16 800 768
+278 353 0
+170 638 6.283185e-01
+137 634 6.457718e-01
+246 556 6.632251e-01
+268 385 7.504916e-01
+239 600 6.981317e-01
+191 623 7.330383e-01
+193 582 8.377580e-01
+241 535 9.424778e-01
+158 598 8.901179e-01
+133 554 1.029744e+00
+187 454 1.064651e+00
+91 521 1.064651e+00
+210 541 1.029744e+00
+294 363 3.455752e+00
+152 445 4.380776e+00

minutiae/MinutiaeNet/Dataset/CoarseNet_train/mnt_files/1.mnt

@@ -0,0 +1,18 @@
+001
+16 800 768
+313 382 3.141593e-01
+353 385 3.665191e-01
+261 384 3.141593e-01
+287 327 4.188790e-01
+356 353 4.014257e-01
+385 197 6.632251e-01
+418 85 6.632251e-01
+397 307 7.679449e-01
+346 302 7.155850e-01
+418 267 7.679449e-01
+418 235 7.155850e-01
+473 233 8.552113e-01
+458 182 3.822271e+00
+349 277 3.787364e+00
+418 277 3.892084e+00
+453 235 3.839724e+00

minutiae/MinutiaeNet/Dataset/CoarseNet_train/mnt_files/2.mnt

@@ -0,0 +1,18 @@
+002
+16 800 768
+278 353 0
+170 638 6.283185e-01
+137 634 6.457718e-01
+246 556 6.632251e-01
+268 385 7.504916e-01
+239 600 6.981317e-01
+191 623 7.330383e-01
+193 582 8.377580e-01
+241 535 9.424778e-01
+158 598 8.901179e-01
+133 554 1.029744e+00
+187 454 1.064651e+00
+91 521 1.064651e+00
+210 541 1.029744e+00
+294 363 3.455752e+00
+152 445 4.380776e+00

minutiae/MinutiaeNet/Demo_notebooks/demo_CoarseNet.ipynb

@@ -0,0 +1,294 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Testing CoarseNet\n",
+    "Code for FineNet in paper \"Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge\" at ICB 2018: https://arxiv.org/pdf/1712.09401.pdf\n",
+    "\n",
+    "If you use whole or partial function in this code, please cite paper:\n",
+    "\n",
+    "    @inproceedings{Nguyen_MinutiaeNet,\n",
+    "\tauthor    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},\n",
+    "\ttitle     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},\n",
+    "\tbooktitle = {The 11th International Conference on Biometrics, 2018},\n",
+    "\tyear      = {2018},\n",
+    "\t}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To run this script, you need to prepare dataset as follows:\n",
+    "`path/to/dataset/`:\n",
+    " - img_files/*.bmp\n",
+    "\n",
+    "If using groundtruth mask instead of mask generated by CoarseNet:\n",
+    " - seg_files/*.bmp\n",
+    " \n",
+    "## CoarseNet can run with any image size\n",
+    "See [CoarseNet_run.py](https://github.com/luannd/MinutiaeNet/blob/master/CoarseNet/CoarseNet_run.py) if running from command line.\n",
+    "\n",
+    "CoarseNet can be improved by:\n",
+    "- Train on new dataset instead of FVC\n",
+    "- Correct the orientation\n",
+    "- Tune threshold for different dataset\n",
+    "\n",
+    "## CoarseNet can provides:\n",
+    "- Orientation field estimation\n",
+    "- Mask for fingerprint area\n",
+    "- Minutiae location and orientation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using TensorFlow backend.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from __future__ import absolute_import\n",
+    "from __future__ import division\n",
+    "\n",
+    "import sys, os\n",
+    "sys.path.append(os.path.realpath('../CoarseNet'))\n",
+    "\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '1'\n",
+    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
+    "\n",
+    "\n",
+    "from keras import backend as K\n",
+    "\n",
+    "from MinutiaeNet_utils import *\n",
+    "from CoarseNet_utils import *\n",
+    "from CoarseNet_model import *\n",
+    "import argparse\n",
+    "\n",
+    "\n",
+    "config = K.tf.ConfigProto(gpu_options=K.tf.GPUOptions(allow_growth=True))\n",
+    "sess = K.tf.Session(config=config)\n",
+    "K.set_session(sess)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Prepare dataset for testing. \n",
+    "inference_set = ['../Dataset/CoarseNet_test/',]\n",
+    "\n",
+    "CoarseNet_path = '../Models/CoarseNet.h5'\n",
+    "\n",
+    "output_dir = '../output_CoarseNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')\n",
+    "\n",
+    "FineNet_path = '../Models/FineNet.h5'\n",
+    "\n",
+    "logging = init_log(output_dir)\n",
+    "\n",
+    "# If use FineNet to refine, set into True\n",
+    "isHavingFineNet = False"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This can test with different folders.\n",
+    "\n",
+    "Threshold for each image is automatically chosen"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "for i, deploy_set in enumerate(inference_set):\n",
+    "    set_name = deploy_set.split('/')[-2]\n",
+    "\n",
+    "    # Read image and GT\n",
+    "    img_name, folder_name, img_size = get_maximum_img_size_and_names(deploy_set)\n",
+    "\n",
+    "    mkdir(output_dir + '/'+ set_name + '/')\n",
+    "    mkdir(output_dir + '/' + set_name + '/mnt_results/')\n",
+    "    mkdir(output_dir + '/'+ set_name + '/seg_results/')\n",
+    "    mkdir(output_dir + '/' + set_name + '/OF_results/')\n",
+    "\n",
+    "    logging.info(\"Predicting \\\"%s\\\":\" % (set_name))\n",
+    "\n",
+    "\n",
+    "    main_net_model = CoarseNetmodel((None, None, 1), CoarseNet_path, mode='deploy')\n",
+    "\n",
+    "    # ====== Load FineNet to verify\n",
+    "    if isHavingFineNet == True:\n",
+    "        model_FineNet = FineNetmodel(num_classes=2,\n",
+    "                             pretrained_path=FineNet_path,\n",
+    "                             input_shape=(224,224,3))\n",
+    "\n",
+    "        model_FineNet.compile(loss='categorical_crossentropy',\n",
+    "                      optimizer=Adam(lr=0),\n",
+    "                      metrics=['accuracy'])\n",
+    "\n",
+    "    for i in xrange(0, len(img_name)):\n",
+    "        \n",
+    "        logging.info(\"\\\"%s\\\" %d / %d: %s\" % (set_name, i + 1, len(img_name), img_name[i]))\n",
+    "\n",
+    "        image = misc.imread(deploy_set + 'img_files/' + img_name[i] + '.bmp', mode='L')# / 255.0\n",
+    "\n",
+    "        img_size = image.shape\n",
+    "        img_size = np.array(img_size, dtype=np.int32) // 8 * 8\n",
+    "        image = image[:img_size[0], :img_size[1]]\n",
+    "\n",
+    "        original_image = image.copy()\n",
+    "\n",
+    "        # Generate OF\n",
+    "        texture_img = FastEnhanceTexture(image, sigma=2.5, show=False)\n",
+    "        dir_map, fre_map = get_maps_STFT(texture_img, patch_size=64, block_size=16, preprocess=True)\n",
+    "        \n",
+    "        image = np.reshape(image, [1, image.shape[0], image.shape[1], 1])\n",
+    "\n",
+    "        enh_img, enh_img_imag, enhance_img, ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out \\\n",
+    "            = main_net_model.predict(image)\n",
+    "\n",
+    "        # Use for output mask\n",
+    "        round_seg = np.round(np.squeeze(seg_out))\n",
+    "        seg_out = 1 - round_seg\n",
+    "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))\n",
+    "        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_CLOSE, kernel)\n",
+    "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7))\n",
+    "        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_OPEN, kernel)\n",
+    "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))\n",
+    "        seg_out = cv2.dilate(seg_out, kernel)\n",
+    "\n",
+    "        #========== Adaptive threshold ==================\n",
+    "        final_minutiae_score_threashold = 0.45\n",
+    "        early_minutiae_thres = final_minutiae_score_threashold + 0.05\n",
+    "\n",
+    "\n",
+    "\n",
+    "        # In cases of small amount of minutiae given, try adaptive threshold\n",
+    "        while final_minutiae_score_threashold >= 0:\n",
+    "            mnt = label2mnt(np.squeeze(mnt_s_out) * np.round(np.squeeze(seg_out)), mnt_w_out, mnt_h_out, mnt_o_out,\n",
+    "                            thresh=early_minutiae_thres)\n",
+    "\n",
+    "            mnt_nms_1 = py_cpu_nms(mnt, 0.5)\n",
+    "            mnt_nms_2 = nms(mnt)\n",
+    "            # Make sure good result is given\n",
+    "            if mnt_nms_1.shape[0] > 4 and mnt_nms_2.shape[0] > 4:\n",
+    "                break\n",
+    "            else:\n",
+    "                final_minutiae_score_threashold = final_minutiae_score_threashold - 0.05\n",
+    "                early_minutiae_thres = early_minutiae_thres - 0.05\n",
+    "\n",
+    "\n",
+    "        mnt_nms = fuse_nms(mnt_nms_1, mnt_nms_2)\n",
+    "\n",
+    "        mnt_nms = mnt_nms[mnt_nms[:, 3] > early_minutiae_thres, :]\n",
+    "        mnt_refined = []\n",
+    "\n",
+    "        if isHavingFineNet == True:\n",
+    "            # ======= Verify using FineNet ============\n",
+    "            patch_minu_radio = 22\n",
+    "            if FineNet_path != None:\n",
+    "                for idx_minu in range(mnt_nms.shape[0]):\n",
+    "                    try:\n",
+    "                        # Extract patch from image\n",
+    "                        x_begin = int(mnt_nms[idx_minu, 1]) - patch_minu_radio\n",
+    "                        y_begin = int(mnt_nms[idx_minu, 0]) - patch_minu_radio\n",
+    "                        patch_minu = original_image[x_begin:x_begin + 2 * patch_minu_radio,\n",
+    "                                     y_begin:y_begin + 2 * patch_minu_radio]\n",
+    "\n",
+    "                        patch_minu = cv2.resize(patch_minu, dsize=(224, 224), interpolation=cv2.INTER_NEAREST)\n",
+    "\n",
+    "                        ret = np.empty((patch_minu.shape[0], patch_minu.shape[1], 3), dtype=np.uint8)\n",
+    "                        ret[:, :, 0] = patch_minu\n",
+    "                        ret[:, :, 1] = patch_minu\n",
+    "                        ret[:, :, 2] = patch_minu\n",
+    "                        patch_minu = ret\n",
+    "                        patch_minu = np.expand_dims(patch_minu, axis=0)\n",
+    "\n",
+    "                        # # Can use class as hard decision\n",
+    "                        # # 0: minu  1: non-minu\n",
+    "                        # [class_Minutiae] = np.argmax(model_FineNet.predict(patch_minu), axis=1)\n",
+    "                        #\n",
+    "                        # if class_Minutiae == 0:\n",
+    "                        #     mnt_refined.append(mnt_nms[idx_minu,:])\n",
+    "\n",
+    "                        # Use soft decision: merge FineNet score with CoarseNet score\n",
+    "                        [isMinutiaeProb] = model_FineNet.predict(patch_minu)\n",
+    "                        isMinutiaeProb = isMinutiaeProb[0]\n",
+    "                        # print isMinutiaeProb\n",
+    "                        tmp_mnt = mnt_nms[idx_minu, :].copy()\n",
+    "                        tmp_mnt[3] = (4*tmp_mnt[3] + isMinutiaeProb) / 5\n",
+    "                        mnt_refined.append(tmp_mnt)\n",
+    "\n",
+    "                    except:\n",
+    "                        mnt_refined.append(mnt_nms[idx_minu, :])\n",
+    "        else:\n",
+    "            mnt_refined = mnt_nms\n",
+    "\n",
+    "        mnt_nms_backup = mnt_nms.copy()\n",
+    "        mnt_nms = np.array(mnt_refined)\n",
+    "\n",
+    "        if mnt_nms.shape[0] > 0:\n",
+    "            mnt_nms = mnt_nms[mnt_nms[:, 3] > final_minutiae_score_threashold, :]\n",
+    "        \n",
+    "        final_mask = ndimage.zoom(np.round(np.squeeze(seg_out)), [8, 8], order=0)\n",
+    "        # Show the orientation\n",
+    "        show_orientation_field(original_image, dir_map + np.pi, mask=final_mask, fname=\"%s/%s/OF_results/%s_OF.jpg\" % (output_dir, set_name, img_name[i]))\n",
+    "\n",
+    "        fuse_minu_orientation(dir_map, mnt_nms, mode=3)\n",
+    "\n",
+    "        time_afterpost = time()\n",
+    "        mnt_writer(mnt_nms, img_name[i], img_size, \"%s/%s/mnt_results/%s.mnt\"%(output_dir, set_name, img_name[i]))\n",
+    "        draw_minutiae(original_image, mnt_nms, \"%s/%s/%s_minu.jpg\"%(output_dir, set_name, img_name[i]),saveimage=True)\n",
+    "\n",
+    "        misc.imsave(\"%s/%s/seg_results/%s_seg.jpg\" % (output_dir, set_name, img_name[i]), final_mask)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

minutiae/MinutiaeNet/Demo_notebooks/train_CoarseNet.ipynb

@@ -0,0 +1,131 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Training CoarseNet\n",
+    "Code for FineNet in paper \"Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge\" at ICB 2018: https://arxiv.org/pdf/1712.09401.pdf\n",
+    "\n",
+    "If you use whole or partial function in this code, please cite paper:\n",
+    "\n",
+    "    @inproceedings{Nguyen_MinutiaeNet,\n",
+    "\tauthor    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},\n",
+    "\ttitle     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},\n",
+    "\tbooktitle = {The 11th International Conference on Biometrics, 2018},\n",
+    "\tyear      = {2018},\n",
+    "\t}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To run this script, you need to prepare dataset as follows:\n",
+    "`path/to/dataset/`:\n",
+    "```Shell\n",
+    " - img_files/*.bmp\n",
+    " - mnt_files/*.mnt\n",
+    " - seg_files/*.jpg\n",
+    "```\n",
+    "See example at `Dataset/CoarseNet_train/` (these images are example from NIST SD27)\n",
+    " \n",
+    "## CoarseNet can run with any image size\n",
+    "See [CoarseNet_train.py](https://github.com/luannd/MinutiaeNet/blob/master/CoarseNet/CoarseNet_train.py) if running from command line.\n",
+    "\n",
+    "Log files, tensorboard, minutiae models can be seen from `output_CoarseNet` folder"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from __future__ import absolute_import\n",
+    "from __future__ import division\n",
+    "\n",
+    "import sys, os\n",
+    "sys.path.append(os.path.realpath('../CoarseNet'))\n",
+    "\n",
+    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
+    "\n",
+    "from datetime import datetime\n",
+    "from MinutiaeNet_utils import *\n",
+    "\n",
+    "from keras import backend as K\n",
+    "from keras.optimizers import SGD, Adam\n",
+    "\n",
+    "from CoarseNet_utils import *\n",
+    "from CoarseNet_model import *\n",
+    "\n",
+    "lr = 0.005\n",
+    "\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'\n",
+    "\n",
+    "config = K.tf.ConfigProto(gpu_options=K.tf.GPUOptions(allow_growth=True))\n",
+    "sess = K.tf.Session(config=config)\n",
+    "K.set_session(sess)\n",
+    "\n",
+    "batch_size = 2\n",
+    "use_multiprocessing = False\n",
+    "input_size = 400\n",
+    "\n",
+    "# Can use multiple folders for training\n",
+    "train_set = ['../Dataset/CoarseNet_train/',]\n",
+    "validate_set = ['../path/to/your/data/',]\n",
+    "\n",
+    "pretrain_dir = '../Models/CoarseNet.h5'\n",
+    "output_dir = '../output_CoarseNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')\n",
+    "FineNet_dir = '../Models/FineNet.h5'\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "output_dir = '../output_CoarseNet/trainResults/' + datetime.now().strftime('%Y%m%d-%H%M%S')\n",
+    "logging = init_log(output_dir)\n",
+    "logging.info(\"Learning rate = %s\", lr)\n",
+    "logging.info(\"Pretrain dir = %s\", pretrain_dir)\n",
+    "\n",
+    "train(input_shape=(input_size, input_size), train_set=train_set, output_dir=output_dir,\n",
+    "      pretrain_dir=pretrain_dir, batch_size=batch_size, test_set=validate_set,\n",
+    "      learning_config=Adam(lr=float(lr), beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=0.9),\n",
+    "      logging=logging)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

minutiae/MinutiaeNet/Demo_notebooks/train_FineNet.ipynb

@@ -0,0 +1,221 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Training FineNet\n",
+    "Code for FineNet in paper \"Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge\" at ICB 2018: https://arxiv.org/pdf/1712.09401.pdf\n",
+    "\n",
+    "If you use whole or partial function in this code, please cite paper:\n",
+    "\n",
+    "    @inproceedings{Nguyen_MinutiaeNet,\n",
+    "\tauthor    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},\n",
+    "\ttitle     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},\n",
+    "\tbooktitle = {The 11th International Conference on Biometrics, 2018},\n",
+    "\tyear      = {2018},\n",
+    "\t}\n",
+    "\n",
+    "Prepare your data as follows:\n",
+    "- Prepare minutiae and non-minutiae image patches with any sizes. I suggest to use `44x44` size\n",
+    "- Put all images in corresponding folers (`minu`, `non_minu`) in \n",
+    "    - `Dataset/train`,\n",
+    "    - `Dataset/test`,\n",
+    "    - `Dataset/validate`.\n",
+    "- Run following code\n",
+    "\n",
+    "Beside running in this notebook, you can run via command line with file [FineNet_train.py](../FineNet/FineNet_train.py)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import sys,os\n",
+    "sys.path.append(os.path.realpath('../FineNet'))\n",
+    "\n",
+    "from keras.optimizers import Adam\n",
+    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard\n",
+    "from keras.callbacks import ReduceLROnPlateau\n",
+    "from keras.preprocessing.image import ImageDataGenerator\n",
+    "from FineNet_model import FineNetmodel, plot_confusion_matrix\n",
+    "\n",
+    "import numpy as np\n",
+    "import os\n",
+    "from sklearn.metrics import confusion_matrix\n",
+    "from datetime import datetime\n",
+    "\n",
+    "\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '2'\n",
+    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
+    "\n",
+    "\n",
+    "output_dir = '../output_FineNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')\n",
+    "\n",
+    "# Prepare model model saving directory.\n",
+    "save_dir = os.path.join(os.getcwd(), output_dir)\n",
+    "log_dir = os.path.join(os.getcwd(), output_dir + '/logs')\n",
+    "\n",
+    "# Training parameters\n",
+    "batch_size = 32\n",
+    "epochs = 200\n",
+    "num_classes = 2\n",
+    "\n",
+    "# Subtracting pixel mean improves accuracy\n",
+    "subtract_pixel_mean = True\n",
+    "\n",
+    "# Model size, patch\n",
+    "model_type = 'patch224batch32'\n",
+    "\n",
+    "\n",
+    "# =============== DATA loading ========================\n",
+    "\n",
+    "train_path = '../Dataset/train/'\n",
+    "test_path = '../Dataset/validate/'\n",
+    "\n",
+    "input_shape = (224, 224, 3)\n",
+    "\n",
+    "# Using data augmentation technique for training\n",
+    "datagen = ImageDataGenerator(\n",
+    "        # set input mean to 0 over the dataset\n",
+    "        featurewise_center=False,\n",
+    "        # set each sample mean to 0\n",
+    "        samplewise_center=False,\n",
+    "        # divide inputs by std of dataset\n",
+    "        featurewise_std_normalization=False,\n",
+    "        # divide each input by its std\n",
+    "        samplewise_std_normalization=False,\n",
+    "        # apply ZCA whitening\n",
+    "        zca_whitening=False,\n",
+    "        # randomly rotate images in the range (deg 0 to 180)\n",
+    "        rotation_range=180,\n",
+    "        # randomly shift images horizontally\n",
+    "        width_shift_range=0.5,\n",
+    "        # randomly shift images vertically\n",
+    "        height_shift_range=0.5,\n",
+    "        # randomly flip images\n",
+    "        horizontal_flip=True,\n",
+    "        # randomly flip images\n",
+    "        vertical_flip=True)\n",
+    "\n",
+    "train_batches = datagen.flow_from_directory(train_path, target_size=(input_shape[0], input_shape[1]), classes=['minu', 'non_minu'], batch_size=batch_size)\n",
+    "# Feed data from directory into batches\n",
+    "test_gen = ImageDataGenerator()\n",
+    "test_batches = test_gen.flow_from_directory(test_path, target_size=(input_shape[0], input_shape[1]), classes=['minu', 'non_minu'], batch_size=batch_size)\n",
+    "\n",
+    "\n",
+    "# =============== end DATA loading ========================\n",
+    "\n",
+    "\n",
+    "\n",
+    "def lr_schedule(epoch):\n",
+    "    \"\"\"Learning Rate Schedule\n",
+    "    \"\"\"\n",
+    "    lr = 0.5e-2\n",
+    "    if epoch > 180:\n",
+    "        lr *= 0.5e-3\n",
+    "    elif epoch > 150:\n",
+    "        lr *= 1e-3\n",
+    "    elif epoch > 60:\n",
+    "        lr *= 5e-2\n",
+    "    elif epoch > 30:\n",
+    "        lr *= 5e-1\n",
+    "    print('Learning rate: ', lr)\n",
+    "    return lr\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "#============== Define model ==================\n",
+    "\n",
+    "model = FineNetmodel(num_classes = num_classes,\n",
+    "                     pretrained_path = '../Models/FineNet.h5',\n",
+    "                     input_shape=input_shape)\n",
+    "\n",
+    "# Save model architecture\n",
+    "#plot_model(model, to_file='./modelFineNet.pdf',show_shapes=True)\n",
+    "\n",
+    "model.compile(loss='categorical_crossentropy',\n",
+    "              optimizer=Adam(lr=lr_schedule(0)),\n",
+    "              metrics=['accuracy'])\n",
+    "#model.summary()\n",
+    "\n",
+    "#============== End define model ==============\n",
+    "\n",
+    "\n",
+    "#============== Other stuffs for loging and parameters ==================\n",
+    "model_name = 'FineNet_%s_model.{epoch:03d}.h5' % model_type\n",
+    "if not os.path.isdir(save_dir):\n",
+    "    os.makedirs(save_dir)\n",
+    "if not os.path.isdir(log_dir):\n",
+    "    os.makedirs(log_dir)\n",
+    "\n",
+    "filepath = os.path.join(save_dir, model_name)\n",
+    "\n",
+    "\n",
+    "# Show in tensorboard\n",
+    "tensorboard = TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=False)\n",
+    "\n",
+    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
+    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
+    "                             monitor='val_acc',\n",
+    "                             verbose=1,\n",
+    "                             save_best_only=True)\n",
+    "\n",
+    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
+    "\n",
+    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
+    "                               cooldown=0,\n",
+    "                               patience=5,\n",
+    "                               min_lr=0.5e-6)\n",
+    "\n",
+    "callbacks = [checkpoint, lr_reducer, lr_scheduler, tensorboard]\n",
+    "\n",
+    "#============== End other stuffs  ==================\n",
+    "\n",
+    "# Begin training\n",
+    "model.fit_generator(train_batches,\n",
+    "                    validation_data=test_batches,\n",
+    "                    epochs=epochs, verbose=1,\n",
+    "                    callbacks=callbacks)\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Plot confusion matrix\n",
+    "score = model.evaluate_generator(test_batches)\n",
+    "print 'Test accuracy:', score[1]\n",
+    "predictions = model.predict_generator(test_batches)\n",
+    "test_labels = test_batches.classes[test_batches.index_array]\n",
+    "\n",
+    "cm = confusion_matrix(test_labels, np.argmax(predictions,axis=1))\n",
+    "cm_plot_labels = ['minu','non_minu']\n",
+    "plot_confusion_matrix(cm, cm_plot_labels, title='Confusion Matrix')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

minutiae/MinutiaeNet/FineNet/FineNet_model.py

@@ -0,0 +1,244 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+from keras.models import Model
+from keras.layers import Activation, AveragePooling2D, BatchNormalization, Concatenate, Conv2D, Dense, GlobalAveragePooling2D
+from keras.layers import Input, Lambda, MaxPooling2D
+from keras.applications.imagenet_utils import _obtain_input_shape
+from keras import backend as K
+import matplotlib.pyplot as plt
+import numpy as np
+import itertools
+
+def preprocess_input(x):
+    """Preprocesses a numpy array encoding a batch of images.
+
+    """
+    return keras.applications.imagenet_utils.preprocess_input(x, mode='tf')
+
+
+def conv2d_bn(x,
+              filters,
+              kernel_size,
+              strides=1,
+              padding='same',
+              activation='relu',
+              use_bias=False,
+              name=None):
+    """Utility function to apply conv + BN.
+
+    """
+    x = Conv2D(filters,
+               kernel_size,
+               strides=strides,
+               padding=padding,
+               use_bias=use_bias,
+               name=name)(x)
+    if not use_bias:
+        bn_axis = 1 if K.image_data_format() == 'channels_first' else 3
+        bn_name = None if name is None else name + '_bn'
+        x = BatchNormalization(axis=bn_axis, scale=False, name=bn_name)(x)
+    if activation is not None:
+        ac_name = None if name is None else name + '_ac'
+        x = Activation(activation, name=ac_name)(x)
+    return x
+
+
+def inception_resnet_block(x, scale, block_type, block_idx, activation='relu'):
+    """Inception-ResNet block.
+
+    """
+    if block_type == 'block35':
+        branch_0 = conv2d_bn(x, 32, 1)
+        branch_1 = conv2d_bn(x, 32, 1)
+        branch_1 = conv2d_bn(branch_1, 32, 3)
+        branch_2 = conv2d_bn(x, 32, 1)
+        branch_2 = conv2d_bn(branch_2, 48, 3)
+        branch_2 = conv2d_bn(branch_2, 64, 3)
+        branches = [branch_0, branch_1, branch_2]
+    elif block_type == 'block17':
+        branch_0 = conv2d_bn(x, 192, 1)
+        branch_1 = conv2d_bn(x, 128, 1)
+        branch_1 = conv2d_bn(branch_1, 160, [1, 7])
+        branch_1 = conv2d_bn(branch_1, 192, [7, 1])
+        branches = [branch_0, branch_1]
+    elif block_type == 'block8':
+        branch_0 = conv2d_bn(x, 192, 1)
+        branch_1 = conv2d_bn(x, 192, 1)
+        branch_1 = conv2d_bn(branch_1, 224, [1, 3])
+        branch_1 = conv2d_bn(branch_1, 256, [3, 1])
+        branches = [branch_0, branch_1]
+    else:
+        raise ValueError('Unknown Inception-ResNet block type. '
+                         'Expects "block35", "block17" or "block8", '
+                         'but got: ' + str(block_type))
+
+    block_name = block_type + '_' + str(block_idx)
+    channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
+    mixed = Concatenate(axis=channel_axis, name=block_name + '_mixed')(branches)
+    up = conv2d_bn(mixed,
+                   K.int_shape(x)[channel_axis],
+                   1,
+                   activation=None,
+                   use_bias=True,
+                   name=block_name + '_conv')
+
+    x = Lambda(lambda inputs, scale: inputs[0] + inputs[1] * scale,
+               output_shape=K.int_shape(x)[1:],
+               arguments={'scale': scale},
+               name=block_name)([x, up])
+    if activation is not None:
+        x = Activation(activation, name=block_name + '_ac')(x)
+    return x
+
+def FineNetmodel(num_classes = 2, pretrained_path = None, input_shape = None):
+    """Create FineNet architecture.
+
+    """
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(
+        input_shape,
+        default_size=299,
+        min_size=139,
+        data_format=K.image_data_format(),
+        require_flatten=False,
+        weights=pretrained_path)
+
+
+    img_input = Input(shape=input_shape)
+
+    # Stem block: 35 x 35 x 192
+    x = conv2d_bn(img_input, 32, 3, strides=2, padding='valid')
+    x = conv2d_bn(x, 32, 3, padding='valid')
+    x = conv2d_bn(x, 64, 3)
+    x = MaxPooling2D(3, strides=2)(x)
+    x = conv2d_bn(x, 80, 1, padding='valid')
+    x = conv2d_bn(x, 192, 3, padding='valid')
+    x = MaxPooling2D(3, strides=2)(x)
+
+    # Mixed 5b (Inception-A block): 35 x 35 x 320
+    branch_0 = conv2d_bn(x, 96, 1)
+    branch_1 = conv2d_bn(x, 48, 1)
+    branch_1 = conv2d_bn(branch_1, 64, 5)
+    branch_2 = conv2d_bn(x, 64, 1)
+    branch_2 = conv2d_bn(branch_2, 96, 3)
+    branch_2 = conv2d_bn(branch_2, 96, 3)
+    branch_pool = AveragePooling2D(3, strides=1, padding='same')(x)
+    branch_pool = conv2d_bn(branch_pool, 64, 1)
+    branches = [branch_0, branch_1, branch_2, branch_pool]
+    channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
+    x = Concatenate(axis=channel_axis, name='mixed_5b')(branches)
+
+    # 10x block35 (Inception-ResNet-A block): 35 x 35 x 320
+    for block_idx in range(1, 11):
+        x = inception_resnet_block(x,
+                                   scale=0.17,
+                                   block_type='block35',
+                                   block_idx=block_idx)
+
+    # Mixed 6a (Reduction-A block): 17 x 17 x 1088
+    branch_0 = conv2d_bn(x, 384, 3, strides=2, padding='valid')
+    branch_1 = conv2d_bn(x, 256, 1)
+    branch_1 = conv2d_bn(branch_1, 256, 3)
+    branch_1 = conv2d_bn(branch_1, 384, 3, strides=2, padding='valid')
+    branch_pool = MaxPooling2D(3, strides=2, padding='valid')(x)
+    branches = [branch_0, branch_1, branch_pool]
+    x = Concatenate(axis=channel_axis, name='mixed_6a')(branches)
+
+    # 20x block17 (Inception-ResNet-B block): 17 x 17 x 1088
+    for block_idx in range(1, 21):
+        x = inception_resnet_block(x,
+                                   scale=0.1,
+                                   block_type='block17',
+                                   block_idx=block_idx)
+
+    # Mixed 7a (Reduction-B block): 8 x 8 x 2080
+    branch_0 = conv2d_bn(x, 256, 1)
+    branch_0 = conv2d_bn(branch_0, 384, 3, strides=2, padding='valid')
+    branch_1 = conv2d_bn(x, 256, 1)
+    branch_1 = conv2d_bn(branch_1, 288, 3, strides=2, padding='valid')
+    branch_2 = conv2d_bn(x, 256, 1)
+    branch_2 = conv2d_bn(branch_2, 288, 3)
+    branch_2 = conv2d_bn(branch_2, 320, 3, strides=2, padding='valid')
+    branch_pool = MaxPooling2D(3, strides=2, padding='valid')(x)
+    branches = [branch_0, branch_1, branch_2, branch_pool]
+    x = Concatenate(axis=channel_axis, name='mixed_7a')(branches)
+
+    # 10x block8 (Inception-ResNet-C block): 8 x 8 x 2080
+    for block_idx in range(1, 10):
+        x = inception_resnet_block(x,
+                                   scale=0.2,
+                                   block_type='block8',
+                                   block_idx=block_idx)
+    x = inception_resnet_block(x,
+                               scale=1.,
+                               activation=None,
+                               block_type='block8',
+                               block_idx=10)
+
+    # Final convolution block: 8 x 8 x 1536
+    x = conv2d_bn(x, 1536, 1, name='conv_7b')
+
+    # Classification block
+    x = GlobalAveragePooling2D(name='avg_pool')(x)
+    x = Dense(num_classes, activation='softmax', name='predictions')(x)
+
+
+    inputs = img_input
+
+    # Create model
+    model = Model(inputs, x, name='FineNet')
+
+    # Load weights
+    if pretrained_path != None:
+        print 'Loading FineNet weights from %s'%(pretrained_path)
+        model.load_weights(pretrained_path)
+
+    return model
+
+def plot_confusion_matrix(cm, classes,
+                          normalize=False,
+                          title='Confusion matrix',
+                          cmap=plt.cm.Blues):
+    """
+    This function prints and plots the confusion matrix.
+    Normalization can be applied by setting `normalize=True`.
+    """
+    plt.imshow(cm, interpolation='nearest', cmap=cmap)
+    plt.title(title)
+    plt.colorbar()
+    tick_marks = np.arange(len(classes))
+    plt.xticks(tick_marks, classes, rotation=45)
+    plt.yticks(tick_marks, classes)
+
+    if normalize:
+        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
+        print("Normalized confusion matrix")
+    else:
+        print('Confusion matrix, without normalization')
+
+    print(cm)
+
+    thresh = cm.max() / 2.
+    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
+        plt.text(j, i, cm[i, j],
+                 horizontalalignment="center",
+                 color="white" if cm[i, j] > thresh else "black")
+
+    plt.tight_layout()
+    plt.ylabel('True label')
+    plt.xlabel('Predicted label')
+    plt.show()

minutiae/MinutiaeNet/FineNet/FineNet_train.py

@@ -0,0 +1,172 @@
+"""Code for FineNet in paper "Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge" at ICB 2018
+  https://arxiv.org/pdf/1712.09401.pdf
+
+  If you use whole or partial function in this code, please cite paper:
+
+  @inproceedings{Nguyen_MinutiaeNet,
+    author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+    title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+    booktitle = {The 11th International Conference on Biometrics, 2018},
+    year      = {2018},
+    }
+"""
+
+import sys,os
+sys.path.append(os.path.realpath('../FineNet'))
+
+from keras.optimizers import Adam
+from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard
+from keras.callbacks import ReduceLROnPlateau
+from keras.preprocessing.image import ImageDataGenerator
+from FineNet_model import FineNetmodel, plot_confusion_matrix
+
+import numpy as np
+import os
+from sklearn.metrics import confusion_matrix
+from datetime import datetime
+
+
+os.environ["CUDA_VISIBLE_DEVICES"] = '2'
+os.environ['KERAS_BACKEND'] = 'tensorflow'
+
+
+output_dir = '../output_FineNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')
+
+# Prepare model model saving directory.
+save_dir = os.path.join(os.getcwd(), output_dir)
+log_dir = os.path.join(os.getcwd(), output_dir + '/logs')
+
+# Training parameters
+batch_size = 32
+epochs = 200
+num_classes = 2
+
+# Subtracting pixel mean improves accuracy
+subtract_pixel_mean = True
+
+# Model size, patch
+model_type = 'patch224batch32'
+
+
+# =============== DATA loading ========================
+
+train_path = '../Dataset/train/'
+test_path = '../Dataset/validate/'
+
+input_shape = (224, 224, 3)
+
+# Using data augmentation technique for training
+datagen = ImageDataGenerator(
+        # set input mean to 0 over the dataset
+        featurewise_center=False,
+        # set each sample mean to 0
+        samplewise_center=False,
+        # divide inputs by std of dataset
+        featurewise_std_normalization=False,
+        # divide each input by its std
+        samplewise_std_normalization=False,
+        # apply ZCA whitening
+        zca_whitening=False,
+        # randomly rotate images in the range (deg 0 to 180)
+        rotation_range=180,
+        # randomly shift images horizontally
+        width_shift_range=0.5,
+        # randomly shift images vertically
+        height_shift_range=0.5,
+        # randomly flip images
+        horizontal_flip=True,
+        # randomly flip images
+        vertical_flip=True)
+
+train_batches = datagen.flow_from_directory(train_path, target_size=(input_shape[0], input_shape[1]), classes=['minu', 'non_minu'], batch_size=batch_size)
+# Feed data from directory into batches
+test_gen = ImageDataGenerator()
+test_batches = test_gen.flow_from_directory(test_path, target_size=(input_shape[0], input_shape[1]), classes=['minu', 'non_minu'], batch_size=batch_size)
+
+
+# =============== end DATA loading ========================
+
+
+
+def lr_schedule(epoch):
+    """Learning Rate Schedule
+    """
+    lr = 0.5e-2
+    if epoch > 180:
+        lr *= 0.5e-3
+    elif epoch > 150:
+        lr *= 1e-3
+    elif epoch > 60:
+        lr *= 5e-2
+    elif epoch > 30:
+        lr *= 5e-1
+    print('Learning rate: ', lr)
+    return lr
+
+
+
+
+#============== Define model ==================
+
+model = FineNetmodel(num_classes = num_classes,
+                     pretrained_path = '../Models/FineNet.h5',
+                     input_shape=input_shape)
+
+# Save model architecture
+#plot_model(model, to_file='./modelFineNet.pdf',show_shapes=True)
+
+model.compile(loss='categorical_crossentropy',
+              optimizer=Adam(lr=lr_schedule(0)),
+              metrics=['accuracy'])
+#model.summary()
+
+#============== End define model ==============
+
+
+#============== Other stuffs for loging and parameters ==================
+model_name = 'FineNet_%s_model.{epoch:03d}.h5' % model_type
+if not os.path.isdir(save_dir):
+    os.makedirs(save_dir)
+if not os.path.isdir(log_dir):
+    os.makedirs(log_dir)
+
+filepath = os.path.join(save_dir, model_name)
+
+
+# Show in tensorboard
+tensorboard = TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=False)
+
+# Prepare callbacks for model saving and for learning rate adjustment.
+checkpoint = ModelCheckpoint(filepath=filepath,
+                             monitor='val_acc',
+                             verbose=1,
+                             save_best_only=True)
+
+lr_scheduler = LearningRateScheduler(lr_schedule)
+
+lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
+                               cooldown=0,
+                               patience=5,
+                               min_lr=0.5e-6)
+
+callbacks = [checkpoint, lr_reducer, lr_scheduler, tensorboard]
+
+#============== End other stuffs  ==================
+
+# Begin training
+model.fit_generator(train_batches,
+                    validation_data=test_batches,
+                    epochs=epochs, verbose=1,
+                    callbacks=callbacks)
+
+
+
+# Plot confusion matrix
+score = model.evaluate_generator(test_batches)
+print 'Test accuracy:', score[1]
+predictions = model.predict_generator(test_batches)
+test_labels = test_batches.classes[test_batches.index_array]
+
+cm = confusion_matrix(test_labels, np.argmax(predictions,axis=1))
+cm_plot_labels = ['minu','non_minu']
+plot_confusion_matrix(cm, cm_plot_labels, title='Confusion Matrix')

minutiae/MinutiaeNet/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Dinh-Luan Nguyen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

minutiae/MinutiaeNet/README.md

@@ -0,0 +1,85 @@
+# Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge
+
+By Dinh-Luan Nguyen, Kai Cao and Anil K.Jain
+
+
+<div align="middle">
+    <img src="assets/Pic1.gif" width="300" hspace="30"/>
+    <img src="assets/Pic2.gif" width="300"/>
+</div>
+
+For precise fingerprint segmentation, let's refer to this paper: [Automatic Latent Fingerprint Segmentation](https://arxiv.org/pdf/1804.09650.pdf)
+### Introduction
+We present the framework called **MinutiaeNet** including CoarseNet and FineNet
+
+![MinutiaeNet](assets/MinutiaeNet.jpg)
+
+**CoarseNet** is a residual learning based convolutional neural network that takes a fingerprint image as initial input, and the corresponding enhanced image, segmentation map, and orientation field (computed by the early stages of CoarseNet) as secondary input to generate the minutiae score map. The minutiae orientation is also estimated by comparing with the fingerprint orientation.
+
+![CoarseNet](assets/CoarseNet.jpg)
+
+**FineNet** is a robust inception-resnet based minutiae classifier. It processes each candidate patch, a square region whose center is the candidate minutiae point, to refine the minutiae score map and approximate minutiae orientation by regression. Final minutiae are the classification results.
+
+We refer reader to read [FineNet_architecture.pdf](assets/FineNet_architecture.pdf) for more details of FineNet.
+
+The repository includes:
+* Source code of Minutiae Net which includes CoarseNet and FineNet.
+* Training code for FineNet and CoarseNet
+* Pre-trained weights for FineNet and CoarseNet
+* Jupyter notebooks to visualize the minutiae detection pipeline at every step
+
+
+### License
+
+MinutiaeNet is released under the MIT License.
+
+### Citing
+
+If you find MinutiaeNet useful in your research, please citing:
+
+    @inproceedings{Nguyen_MinutiaeNet,
+	author    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},
+	title     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},
+	booktitle = {The 11th International Conference on Biometrics, 2018},
+	year      = {2018},
+	}
+
+
+### Contents
+1. [Requirements: software](#requirements-software)
+2. [Installation](#installation)
+3. [Demo](#demo)
+4. [Usage](#usage)
+
+# Requirements: software
+`Python 2.7`, `Tensorflow 1.7.0`, `Keras 2.1.6`.
+
+# Installation
+To make life easier, I suggest to use Anaconda for easy installation. Version using pip is similar.
+  ```Shell
+  conda install cv2, numpy, scipy, matplotlib, pydot, graphviz
+  ```
+Download models and put into `Models` folder.
+- **CoarseNet**: [Googledrive](https://drive.google.com/file/d/1alvw_kAyY4sxdzAkGABQR7waux-rgJKm/view?usp=sharing)    ||    [Dropbox](https://www.dropbox.com/s/gppil4wybdjcihy/CoarseNet.h5?dl=0)
+- **FineNet**: [Googledrive](https://drive.google.com/file/d/1wdGZKNNDAyN-fajjVKJoiyDtXAvl-4zq/view?usp=sharing)    ||    [Dropbox](https://www.dropbox.com/s/k7q2vs9255jf2dh/FineNet.h5?dl=0)
+
+
+
+# Demo
+To help understanding MinutiaeNet, there are 2 notebooks for you to play around:
+ - Understanding CoarseNet: [demo_CoarseNet.ipynb](Demo_notebooks/demo_CoarseNet.ipynb)
+ - Understanding FineNet: [demo_FineNet.ipynb](Demo_notebooks/demo_FineNet.ipynb)
+ - MinutiaeNet - a combination of CoarseNet and FineNet: set `isHavingFineNet = False` in CoarsetNet if you want to fuse results from CoarseNet and FineNet
+ 
+# Usage
+ - **FineNet**
+* [demo_FineNet.ipynb](Demo_notebooks/demo_FineNet.ipynb) is useful if you want to integrate into existing minutiae dectection framework/SDKs. It shows an example of using a pre-trained model to verify the detection in your own images.
+* [train_FineNet.ipynb](Demo_notebooks/train_FineNet.ipynb) shows how to train FineNet on your own dataset. 
+
+
+ - **CoarseNet**
+* [demo_CoarseNet.ipynb](Demo_notebooks/demo_CoarseNet.ipynb) can be called to generate minutiae as well as masks and orientation.
+* [train_CoarseNet.ipynb](Demo_notebooks/train_CoarseNet.ipynb) shows how to train CoarseNet on your own dataset. 
+
+Python files which can run directly from command line are also provided.
+Note that models as well as architectures here are slightly different from the paper because of the continuing development of this project

minutiae/main.ipynb

@@ -0,0 +1,294 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Testing CoarseNet\n",
+    "Code for FineNet in paper \"Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge\" at ICB 2018: https://arxiv.org/pdf/1712.09401.pdf\n",
+    "\n",
+    "If you use whole or partial function in this code, please cite paper:\n",
+    "\n",
+    "    @inproceedings{Nguyen_MinutiaeNet,\n",
+    "\tauthor    = {Dinh-Luan Nguyen and Kai Cao and Anil K. Jain},\n",
+    "\ttitle     = {Robust Minutiae Extractor: Integrating Deep Networks and Fingerprint Domain Knowledge},\n",
+    "\tbooktitle = {The 11th International Conference on Biometrics, 2018},\n",
+    "\tyear      = {2018},\n",
+    "\t}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To run this script, you need to prepare dataset as follows:\n",
+    "`path/to/dataset/`:\n",
+    " - img_files/*.bmp\n",
+    "\n",
+    "If using groundtruth mask instead of mask generated by CoarseNet:\n",
+    " - seg_files/*.bmp\n",
+    " \n",
+    "## CoarseNet can run with any image size\n",
+    "See [CoarseNet_run.py](https://github.com/luannd/MinutiaeNet/blob/master/CoarseNet/CoarseNet_run.py) if running from command line.\n",
+    "\n",
+    "CoarseNet can be improved by:\n",
+    "- Train on new dataset instead of FVC\n",
+    "- Correct the orientation\n",
+    "- Tune threshold for different dataset\n",
+    "\n",
+    "## CoarseNet can provides:\n",
+    "- Orientation field estimation\n",
+    "- Mask for fingerprint area\n",
+    "- Minutiae location and orientation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using TensorFlow backend.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from __future__ import absolute_import\n",
+    "from __future__ import division\n",
+    "\n",
+    "import sys, os\n",
+    "sys.path.append(os.path.realpath('../CoarseNet'))\n",
+    "\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '1'\n",
+    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
+    "\n",
+    "\n",
+    "from keras import backend as K\n",
+    "\n",
+    "from MinutiaeNet_utils import *\n",
+    "from CoarseNet_utils import *\n",
+    "from CoarseNet_model import *\n",
+    "import argparse\n",
+    "\n",
+    "\n",
+    "config = K.tf.ConfigProto(gpu_options=K.tf.GPUOptions(allow_growth=True))\n",
+    "sess = K.tf.Session(config=config)\n",
+    "K.set_session(sess)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Prepare dataset for testing. \n",
+    "inference_set = ['../Dataset/CoarseNet_test/',]\n",
+    "\n",
+    "CoarseNet_path = '../Models/CoarseNet.h5'\n",
+    "\n",
+    "output_dir = '../output_CoarseNet/'+datetime.now().strftime('%Y%m%d-%H%M%S')\n",
+    "\n",
+    "FineNet_path = '../Models/FineNet.h5'\n",
+    "\n",
+    "logging = init_log(output_dir)\n",
+    "\n",
+    "# If use FineNet to refine, set into True\n",
+    "isHavingFineNet = False"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This can test with different folders.\n",
+    "\n",
+    "Threshold for each image is automatically chosen"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "for i, deploy_set in enumerate(inference_set):\n",
+    "    set_name = deploy_set.split('/')[-2]\n",
+    "\n",
+    "    # Read image and GT\n",
+    "    img_name, folder_name, img_size = get_maximum_img_size_and_names(deploy_set)\n",
+    "\n",
+    "    mkdir(output_dir + '/'+ set_name + '/')\n",
+    "    mkdir(output_dir + '/' + set_name + '/mnt_results/')\n",
+    "    mkdir(output_dir + '/'+ set_name + '/seg_results/')\n",
+    "    mkdir(output_dir + '/' + set_name + '/OF_results/')\n",
+    "\n",
+    "    logging.info(\"Predicting \\\"%s\\\":\" % (set_name))\n",
+    "\n",
+    "\n",
+    "    main_net_model = CoarseNetmodel((None, None, 1), CoarseNet_path, mode='deploy')\n",
+    "\n",
+    "    # ====== Load FineNet to verify\n",
+    "    if isHavingFineNet == True:\n",
+    "        model_FineNet = FineNetmodel(num_classes=2,\n",
+    "                             pretrained_path=FineNet_path,\n",
+    "                             input_shape=(224,224,3))\n",
+    "\n",
+    "        model_FineNet.compile(loss='categorical_crossentropy',\n",
+    "                      optimizer=Adam(lr=0),\n",
+    "                      metrics=['accuracy'])\n",
+    "\n",
+    "    for i in xrange(0, len(img_name)):\n",
+    "        \n",
+    "        logging.info(\"\\\"%s\\\" %d / %d: %s\" % (set_name, i + 1, len(img_name), img_name[i]))\n",
+    "\n",
+    "        image = misc.imread(deploy_set + 'img_files/' + img_name[i] + '.bmp', mode='L')# / 255.0\n",
+    "\n",
+    "        img_size = image.shape\n",
+    "        img_size = np.array(img_size, dtype=np.int32) // 8 * 8\n",
+    "        image = image[:img_size[0], :img_size[1]]\n",
+    "\n",
+    "        original_image = image.copy()\n",
+    "\n",
+    "        # Generate OF\n",
+    "        texture_img = FastEnhanceTexture(image, sigma=2.5, show=False)\n",
+    "        dir_map, fre_map = get_maps_STFT(texture_img, patch_size=64, block_size=16, preprocess=True)\n",
+    "        \n",
+    "        image = np.reshape(image, [1, image.shape[0], image.shape[1], 1])\n",
+    "\n",
+    "        enh_img, enh_img_imag, enhance_img, ori_out_1, ori_out_2, seg_out, mnt_o_out, mnt_w_out, mnt_h_out, mnt_s_out \\\n",
+    "            = main_net_model.predict(image)\n",
+    "\n",
+    "        # Use for output mask\n",
+    "        round_seg = np.round(np.squeeze(seg_out))\n",
+    "        seg_out = 1 - round_seg\n",
+    "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 10))\n",
+    "        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_CLOSE, kernel)\n",
+    "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7))\n",
+    "        seg_out = cv2.morphologyEx(seg_out, cv2.MORPH_OPEN, kernel)\n",
+    "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))\n",
+    "        seg_out = cv2.dilate(seg_out, kernel)\n",
+    "\n",
+    "        #========== Adaptive threshold ==================\n",
+    "        final_minutiae_score_threashold = 0.45\n",
+    "        early_minutiae_thres = final_minutiae_score_threashold + 0.05\n",
+    "\n",
+    "\n",
+    "\n",
+    "        # In cases of small amount of minutiae given, try adaptive threshold\n",
+    "        while final_minutiae_score_threashold >= 0:\n",
+    "            mnt = label2mnt(np.squeeze(mnt_s_out) * np.round(np.squeeze(seg_out)), mnt_w_out, mnt_h_out, mnt_o_out,\n",
+    "                            thresh=early_minutiae_thres)\n",
+    "\n",
+    "            mnt_nms_1 = py_cpu_nms(mnt, 0.5)\n",
+    "            mnt_nms_2 = nms(mnt)\n",
+    "            # Make sure good result is given\n",
+    "            if mnt_nms_1.shape[0] > 4 and mnt_nms_2.shape[0] > 4:\n",
+    "                break\n",
+    "            else:\n",
+    "                final_minutiae_score_threashold = final_minutiae_score_threashold - 0.05\n",
+    "                early_minutiae_thres = early_minutiae_thres - 0.05\n",
+    "\n",
+    "\n",
+    "        mnt_nms = fuse_nms(mnt_nms_1, mnt_nms_2)\n",
+    "\n",
+    "        mnt_nms = mnt_nms[mnt_nms[:, 3] > early_minutiae_thres, :]\n",
+    "        mnt_refined = []\n",
+    "\n",
+    "        if isHavingFineNet == True:\n",
+    "            # ======= Verify using FineNet ============\n",
+    "            patch_minu_radio = 22\n",
+    "            if FineNet_path != None:\n",
+    "                for idx_minu in range(mnt_nms.shape[0]):\n",
+    "                    try:\n",
+    "                        # Extract patch from image\n",
+    "                        x_begin = int(mnt_nms[idx_minu, 1]) - patch_minu_radio\n",
+    "                        y_begin = int(mnt_nms[idx_minu, 0]) - patch_minu_radio\n",
+    "                        patch_minu = original_image[x_begin:x_begin + 2 * patch_minu_radio,\n",
+    "                                     y_begin:y_begin + 2 * patch_minu_radio]\n",
+    "\n",
+    "                        patch_minu = cv2.resize(patch_minu, dsize=(224, 224), interpolation=cv2.INTER_NEAREST)\n",
+    "\n",
+    "                        ret = np.empty((patch_minu.shape[0], patch_minu.shape[1], 3), dtype=np.uint8)\n",
+    "                        ret[:, :, 0] = patch_minu\n",
+    "                        ret[:, :, 1] = patch_minu\n",
+    "                        ret[:, :, 2] = patch_minu\n",
+    "                        patch_minu = ret\n",
+    "                        patch_minu = np.expand_dims(patch_minu, axis=0)\n",
+    "\n",
+    "                        # # Can use class as hard decision\n",
+    "                        # # 0: minu  1: non-minu\n",
+    "                        # [class_Minutiae] = np.argmax(model_FineNet.predict(patch_minu), axis=1)\n",
+    "                        #\n",
+    "                        # if class_Minutiae == 0:\n",
+    "                        #     mnt_refined.append(mnt_nms[idx_minu,:])\n",
+    "\n",
+    "                        # Use soft decision: merge FineNet score with CoarseNet score\n",
+    "                        [isMinutiaeProb] = model_FineNet.predict(patch_minu)\n",
+    "                        isMinutiaeProb = isMinutiaeProb[0]\n",
+    "                        # print isMinutiaeProb\n",
+    "                        tmp_mnt = mnt_nms[idx_minu, :].copy()\n",
+    "                        tmp_mnt[3] = (4*tmp_mnt[3] + isMinutiaeProb) / 5\n",
+    "                        mnt_refined.append(tmp_mnt)\n",
+    "\n",
+    "                    except:\n",
+    "                        mnt_refined.append(mnt_nms[idx_minu, :])\n",
+    "        else:\n",
+    "            mnt_refined = mnt_nms\n",
+    "\n",
+    "        mnt_nms_backup = mnt_nms.copy()\n",
+    "        mnt_nms = np.array(mnt_refined)\n",
+    "\n",
+    "        if mnt_nms.shape[0] > 0:\n",
+    "            mnt_nms = mnt_nms[mnt_nms[:, 3] > final_minutiae_score_threashold, :]\n",
+    "        \n",
+    "        final_mask = ndimage.zoom(np.round(np.squeeze(seg_out)), [8, 8], order=0)\n",
+    "        # Show the orientation\n",
+    "        show_orientation_field(original_image, dir_map + np.pi, mask=final_mask, fname=\"%s/%s/OF_results/%s_OF.jpg\" % (output_dir, set_name, img_name[i]))\n",
+    "\n",
+    "        fuse_minu_orientation(dir_map, mnt_nms, mode=3)\n",
+    "\n",
+    "        time_afterpost = time()\n",
+    "        mnt_writer(mnt_nms, img_name[i], img_size, \"%s/%s/mnt_results/%s.mnt\"%(output_dir, set_name, img_name[i]))\n",
+    "        draw_minutiae(original_image, mnt_nms, \"%s/%s/%s_minu.jpg\"%(output_dir, set_name, img_name[i]),saveimage=True)\n",
+    "\n",
+    "        misc.imsave(\"%s/%s/seg_results/%s_seg.jpg\" % (output_dir, set_name, img_name[i]), final_mask)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}