Keras 中Leaky ReLU等高级激活函数的用法

from keras import layers
from keras import models
 
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1))) 
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu')) 
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))

from keras import layers
from keras import models
from keras.layers import LeakyReLU
 
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), input_shape=(28, 28, 1)))
model.add(LeakyReLU(alpha=0.05))
model.add(layers.MaxPooling2D((2, 2))) 
 
model.add(layers.Conv2D(64, (3, 3))) 
model.add(LeakyReLU(alpha=0.05))
model.add(layers.MaxPooling2D((2, 2)))
 
model.add(layers.Conv2D(64, (3, 3))
model.add(LeakyReLU(alpha=0.05))

def relu(x, alpha=0., max_value=None, threshold=0):
 """Rectified linear unit.
 With default values, it returns element-wise `max(x, 0)`.
 Otherwise, it follows:
 `f(x) = max_value` for `x >= max_value`,
 `f(x) = x` for `threshold <= x < max_value`,
 `f(x) = alpha * (x - threshold)` otherwise.
 Arguments:
   x: A tensor or variable.
   alpha: A scalar, slope of negative section (default=`0.`).
   max_value: float. Saturation threshold.
   threshold: float. Threshold value for thresholded activation.
 Returns:
   A tensor.
 """

 if alpha != 0.:
  if max_value is None and threshold == 0:
   return nn.leaky_relu(x, alpha=alpha)  ##在这里调用了leaky_relu

  if threshold != 0:
   negative_part = nn.relu(-x + threshold)
  else:
   negative_part = nn.relu(-x)

 clip_max = max_value is not None

 if threshold != 0:
  # computes x for x > threshold else 0
  x = x * math_ops.cast(math_ops.greater(x, threshold), floatx())
 elif max_value == 6:
  # if no threshold, then can use nn.relu6 native TF op for performance
  x = nn.relu6(x)
  clip_max = False
 else:
  x = nn.relu(x)

 if clip_max:
  max_value = _constant_to_tensor(max_value, x.dtype.base_dtype)
  zero = _constant_to_tensor(0, x.dtype.base_dtype)
  x = clip_ops.clip_by_value(x, zero, max_value)

 if alpha != 0.:
  alpha = _to_tensor(alpha, x.dtype.base_dtype)
  x -= alpha * negative_part
 return x