# -*- coding: utf-8 -*-
""" Deeplabv3+ model for Keras.
This model is based on TF repo:
https://github.com/tensorflow/models/tree/master/research/deeplab
On Pascal VOC, original model gets to 84.56% mIOU
MobileNetv2 backbone is based on this repo:
https://github.com/JonathanCMitchell/mobilenet_v2_keras
# Reference
- [Encoder-Decoder with Atrous Separable Convolution
for Semantic Image Segmentation](https://arxiv.org/pdf/1802.02611.pdf)
- [Xception: Deep Learning with Depthwise Separable Convolutions]
(https://arxiv.org/abs/1610.02357)
- [Inverted Residuals and Linear Bottlenecks: Mobile Networks for
Classification, Detection and Segmentation](https://arxiv.org/abs/1801.04381)
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
from keras.models import Model,load_model
from keras import layers
from keras.layers import Input,Activation,Concatenate,Add,Dropout,BatchNormalization,Conv2D,MaxPooling2D,UpSampling2D
from keras.layers import DepthwiseConv2D,ZeroPadding2D,AveragePooling2D
from keras.engine import Layer,InputSpec
from keras.engine.topology import get_source_inputs
from keras.optimizers import Adam
from keras import backend as K
from keras.applications import imagenet_utils
from keras.utils import conv_utils,plot_model
from keras.utils.data_utils import get_file
#WEIGHTS_PATH_X = "https://github.com/bonlime/keras-deeplab-v3-plus/releases/download/1.1/deeplabv3_xception_tf_dim_ordering_tf_kernels.h5"
#WEIGHTS_PATH_MOBILE = "https://github.com/bonlime/keras-deeplab-v3-plus/releases/download/1.1/deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5"
smooth = 1e-12
class BilinearUpsampling(Layer):
"""Just a simple bilinear upsampling layer. Works only with TF.
Args:
upsampling: tuple of 2 numbers > 0. The upsampling ratio for h and w
output_size: used instead of upsampling arg if passed!
"""
def __init__(self, upsampling=(2, 2), output_size=None, data_format=None, **kwargs):
super(BilinearUpsampling, self).__init__(**kwargs)
self.data_format = conv_utils.normalize_data_format(data_format)
self.input_spec = InputSpec(ndim=4)
if output_size:
self.output_size = conv_utils.normalize_tuple(output_size, 2, 'output_size')
self.upsampling = None
else:
self.output_size = None
self.upsampling = conv_utils.normalize_tuple(upsampling, 2, 'upsampling')
def compute_output_shape(self, input_shape):
if self.upsampling:
height = self.upsampling[0] * input_shape[1] if input_shape[1] is not None else None
width = self.upsampling[1] * input_shape[2] if input_shape[2] is not None else None
else:
height = self.output_size[0]
width = self.output_size[1]
return (input_shape[0],height,width,input_shape[3])
def call(self, inputs):
if self.upsampling:
return K.tf.image.resize_bilinear(inputs, (inputs.shape[1] * self.upsampling[0],inputs.shape[2] * self.upsampling[1]),align_corners=True)
else:
return K.tf.image.resize_bilinear(inputs, (self.output_size[0],self.output_size[1]),align_corners=True)
def get_config(self):
config = {'upsampling': self.upsampling,'output_size': self.output_size,'data_format': self.data_format}
base_config = super(BilinearUpsampling, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def SepConv_BN(x, filters, prefix, stride=1, kernel_size=3, rate=1, depth_activation=False, epsilon=1e-3):
""" SepConv with BN between depthwise & pointwise. Optionally add activation after BN
Implements right "same" padding for even kernel sizes
Args:
x: input tensor
filters: num of filters in pointwise convolution
prefix: prefix before name
stride: stride at depthwise conv
kernel_size: kernel size for depthwise convolution
rate: atrous rate for depthwise convolution
depth_activation: flag to use activation between depthwise & poinwise convs
epsilon: epsilon to use in BN layer
"""
if stride == 1:
depth_padding = 'same'
else:
kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)
pad_total = kernel_size_effective - 1
pad_beg = pad_total // 2
pad_end = pad_total - pad_beg
x = ZeroPadding2D((pad_beg, pad_end))(x)
depth_padding = 'valid'
if not depth_activation:
x = Activation('relu')(x)
x = DepthwiseConv2D((kernel_size, kernel_size), strides=(stride, stride), dilation_rate=(rate, rate),
padding=depth_padding, use_bias=False, name=prefix + '_depthwise')(x)
x = BatchNormalization(name=prefix + '_depthwise_BN', epsilon=epsilon)(x)
if depth_activation:
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1), padding='same', use_bias=False, name=prefix + '_pointwise')(x)
x = BatchNormalization(name=prefix + '_pointwise_BN', epsilon=epsilon)(x)
if depth_activation:
x = Activation('relu')(x)
return x
def jaccard_coef(y_true, y_pred):
intersection = K.sum(y_true * y_pred, axis=[0, -1, -2])
sum_ = K.sum(y_true + y_pred, axis=[0, -1, -2])
jac = (intersection + smooth) / (sum_ - intersection + smooth)
jac_mean=K.mean(jac)
print(jac_mean)
return jac_mean
def _conv2d_same(x, filters, prefix, stride=1, kernel_size=3, rate=1):
"""Implements right 'same' padding for even kernel sizes
Without this there is a 1 pixel drift when stride = 2
Args:
x: input tensor
filters: num of filters in pointwise convolution
prefix: prefix before name
stride: stride at depthwise conv
kernel_size: kernel size for depthwise convolution
rate: atrous rate for depthwise convolution
"""
if stride == 1:
return Conv2D(filters,(kernel_size, kernel_size),strides=(stride, stride),
padding='same', use_bias=False,dilation_rate=(rate, rate),name=prefix)(x)
else:
kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)
pad_total = kernel_size_effective - 1
pad_beg = pad_total // 2
pad_end = pad_total - pad_beg
x = ZeroPadding2D((pad_beg, pad_end))(x)
return Conv2D(filters,(kernel_size, kernel_size),strides=(stride, stride),
padding='valid', use_bias=False,dilation_rate=(rate, rate),name=prefix)(x)
def _xception_block(inputs, depth_list, prefix, skip_connection_type, stride,rate=1, depth_activation=False, return_skip=False):
""" Basic building block of modified Xception network
Args:
inputs: input tensor
depth_list: number of filters in each SepConv layer. len(depth_list) == 3
prefix: prefix before name
skip_connection_type: one of {'conv','sum','none'}
stride: stride at last depthwise conv
rate: atrous rate for depthwise convolution
depth_activation: flag to use activation between depthwise & pointwise convs
return_skip: flag to return additional tensor after 2 SepConvs for decoder
"""
residual = inputs
for i in range(3):
residual = SepConv_BN(residual,depth_list[i],prefix + '_separable_conv{}'.format(i + 1),
stride=stride if i == 2 else 1,rate=rate,depth_activation=depth_activation)
if i == 1:
skip = residual
if skip_connection_type == 'conv':
shortcut = _conv2d_same(inputs, depth_list[-1], prefix + '_shortcut',kernel_size=1,stride=stride)
shortcut = BatchNormalization(name=prefix + '_shortcut_BN')(shortcut)
outputs = layers.add([residual, shortcut])
elif skip_connection_type == 'sum':