L2池化，随机池化，最大池化，平均池化各自的提出时间

L2池化（也称为欧式池化）的提出时间不是很明确，但通常认为是在2012年AlexNet网络结构中首次使用。

随机池化是在2013年的一篇论文中提出的，论文名为"Improving neural networks by preventing co-adaptation of feature detectors"。

最大池化的提出时间可以追溯到1990年代，但真正流行起来是在2012年AlexNet网络结构中的广泛使用。

平均池化也是在早期的网络结构中使用的，但具体的提出时间不是很明确。

相关问题

基于tensorflow2.5,将Channel Attention部分中的全局平均池化替换为L2池化，对于Spatial Attention，将max pooling和average pooling分别替换为随机池化和L2池化，二者结合形成新的CBAM模块，并给出使用示例

首先，我们需要定义L2池化和随机池化的函数：

import tensorflow as tf

def l2_pool(x, pool_size):
    return tf.math.sqrt(tf.nn.avg_pool(tf.square(x), pool_size, strides=1, padding='SAME'))

def rand_pool(x, pool_size):
    random_mat = tf.random.uniform(x.shape[:-1], dtype=tf.float32)
    threshold = tf.math.reduce_max(random_mat) * 0.5
    return tf.where(random_mat > threshold, tf.nn.avg_pool(x, pool_size, strides=1, padding='SAME'), tf.nn.max_pool(x, pool_size, strides=1, padding='SAME'))

然后，我们可以定义新的CBAM模块：

import tensorflow as tf

class NewCBAM(tf.keras.layers.Layer):
    def __init__(self, reduction_ratio=16):
        super(NewCBAM, self).__init__()
        self.reduction_ratio = reduction_ratio
        self.gamma = tf.Variable(0.0, trainable=True)
        
        # Channel Attention
        self.avg_pool = tf.keras.layers.GlobalAveragePooling2D()
        self.max_pool = tf.keras.layers.GlobalMaxPooling2D()
        self.l2_pool = tf.keras.layers.Lambda(lambda x: l2_pool(x, pool_size=(1,1)))
        self.dense1 = tf.keras.layers.Dense(units=tf.keras.backend.int_shape(x)[-1]//self.reduction_ratio, activation='relu')
        self.dense2 = tf.keras.layers.Dense(units=tf.keras.backend.int_shape(x)[-1], activation='sigmoid')
        
        # Spatial Attention
        self.rand_pool = tf.keras.layers.Lambda(lambda x: rand_pool(x, pool_size=(3,3)))
        self.l2_pool2 = tf.keras.layers.Lambda(lambda x: l2_pool(x, pool_size=(3,3)))
        self.concat1 = tf.keras.layers.Concatenate(axis=-1)
        self.conv1 = tf.keras.layers.Conv2D(filters=1, kernel_size=1, activation='sigmoid')
        
    def call(self, x):
        # Channel Attention
        avg_pool = self.avg_pool(x)
        max_pool = self.max_pool(x)
        l2_pool = self.l2_pool(x)
        concat = tf.concat([avg_pool, max_pool, l2_pool], axis=-1)
        dense1 = self.dense1(concat)
        dense2 = self.dense2(dense1)
        channel_attention = tf.reshape(dense2, shape=(-1,1,1,tf.keras.backend.int_shape(x)[-1]))
        channel_attention = tf.math.multiply(x, channel_attention)
        
        # Spatial Attention
        rand_pool = self.rand_pool(x)
        l2_pool2 = self.l2_pool2(x)
        concat2 = tf.concat([rand_pool, l2_pool2], axis=-1)
        conv1 = self.conv1(concat2)
        spatial_attention = tf.math.multiply(x, conv1)
        
        # Fusion
        fusion = tf.math.add(channel_attention, spatial_attention)
        return tf.math.add(tf.math.multiply(x, 1-self.gamma), tf.math.multiply(fusion, self.gamma))

最后，我们可以使用新的CBAM模块：

import tensorflow as tf

inputs = tf.keras.layers.Input(shape=(224, 224, 3))
x = tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding='SAME')(inputs)
x = NewCBAM()(x)
x = tf.keras.layers.Conv2D(filters=128, kernel_size=3, padding='SAME')(x)
x = NewCBAM()(x)
x = tf.keras.layers.Conv2D(filters=256, kernel_size=3, padding='SAME')(x)
x = NewCBAM()(x)
x = tf.keras.layers.Conv2D(filters=512, kernel_size=3, padding='SAME')(x)
x = NewCBAM()(x)
x = tf.keras.layers.Conv2D(filters=1024, kernel_size=3, padding='SAME')(x)
x = NewCBAM()(x)
outputs = tf.keras.layers.GlobalAveragePooling2D()(x)

model = tf.keras.Model(inputs=inputs, outputs=outputs)

基于tensorflow2.5,使用随机池化和L2池化替换CBAM注意力机制模块里的池化操作，并给出使用示例

CBAM注意力机制模块中的池化操作包括全局最大池化和全局平均池化。我们可以使用随机池化和L2池化来替换这些操作。

随机池化是一种随机抽样的池化方法，可以在一定程度上增加模型的鲁棒性和泛化能力。L2池化可以有效地压缩信息，减少模型的计算复杂度和参数数量。

下面是使用随机池化和L2池化替换CBAM注意力机制模块里的池化操作的示例代码：

import tensorflow as tf
from tensorflow.keras.layers import Conv2D, Dense, GlobalAveragePooling2D, GlobalMaxPooling2D, Reshape, Multiply, Add, Activation, Lambda
from tensorflow.keras import backend as K

class CBAM(tf.keras.Model):
    def __init__(self, reduction_ratio=0.5):
        super().__init__()
        self.reduction_ratio = reduction_ratio

    def build(self, input_shape):
        self.channel_axis = -1
        self.max_pool = tf.keras.layers.Lambda(lambda x: tf.reduce_max(x, axis=[1,2], keepdims=True))
        self.avg_pool = tf.keras.layers.Lambda(lambda x: tf.reduce_mean(x, axis=[1,2], keepdims=True))
        self.shared_layer_one = tf.keras.layers.Dense(units=int(input_shape[self.channel_axis])//self.reduction_ratio, activation='relu', kernel_initializer='he_normal', use_bias=True, bias_initializer='zeros')
        self.shared_layer_two = tf.keras.layers.Dense(units=int(input_shape[self.channel_axis]), kernel_initializer='he_normal', use_bias=True, bias_initializer='zeros')
        self.sigmoid_gamma = tf.keras.layers.Activation('sigmoid')

    def call(self, inputs):
        max_pool = self.max_pool(inputs)
        avg_pool = self.avg_pool(inputs)

        gamma = self.shared_layer_two(self.shared_layer_one(tf.concat([max_pool, avg_pool], axis=self.channel_axis)))
        gamma = self.sigmoid_gamma(gamma)

        return inputs * gamma

使用随机池化和L2池化替换CBAM注意力机制模块中的池化操作的代码如下：

import tensorflow as tf
from tensorflow.keras.layers import Conv2D, Dense, GlobalAveragePooling2D, GlobalMaxPooling2D, Reshape, Multiply, Add, Activation, Lambda
from tensorflow.keras import backend as K

class CBAM(tf.keras.Model):
    def __init__(self, reduction_ratio=0.5):
        super().__init__()
        self.reduction_ratio = reduction_ratio

    def build(self, input_shape):
        self.channel_axis = -1
        self.random_pool = tf.keras.layers.Lambda(lambda x: tf.math.reduce_mean(tf.random.shuffle(x), axis=[1,2], keepdims=True))
        self.l2_pool = tf.keras.layers.Lambda(lambda x: tf.math.sqrt(tf.math.reduce_mean(tf.math.square(x), axis=[1,2], keepdims=True)))
        self.shared_layer_one = tf.keras.layers.Dense(units=int(input_shape[self.channel_axis])//self.reduction_ratio, activation='relu', kernel_initializer='he_normal', use_bias=True, bias_initializer='zeros')
        self.shared_layer_two = tf.keras.layers.Dense(units=int(input_shape[self.channel_axis]), kernel_initializer='he_normal', use_bias=True, bias_initializer='zeros')
        self.sigmoid_gamma = tf.keras.layers.Activation('sigmoid')

    def call(self, inputs):
        random_pool = self.random_pool(inputs)
        l2_pool = self.l2_pool(inputs)

        gamma = self.shared_layer_two(self.shared_layer_one(tf.concat([random_pool, l2_pool], axis=self.channel_axis)))
        gamma = self.sigmoid_gamma(gamma)

        return inputs * gamma

使用示例：

import tensorflow as tf
from cbam import CBAM

inputs = tf.keras.layers.Input(shape=(224,224,3))
x = tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), padding='same', activation='relu')(inputs)
x = CBAM()(x)
x = tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), padding='same', activation='relu')(x)
x = CBAM()(x)
x = tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), padding='same', activation='relu')(x)
x = CBAM()(x)
x = tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), padding='same', activation='relu')(x)
x = CBAM()(x)
x = tf.keras.layers.GlobalAveragePooling2D()(x)
outputs = tf.keras.layers.Dense(units=10, activation='softmax')(x)

model = tf.keras.Model(inputs, outputs)
model.summary()