Caffe源代码之卷积层和反卷积层

标签[空格] ： Caffe源代码

卷积层是深度学习核心中的核心，反卷积网络则是广泛应用于图像分割领域。在Caffe中，卷积和反卷积是颠倒的，卷积层的前向传播与反卷积层的反向传播原理是一样的；反卷积层的反向传播和卷积层的前向传播是一样的。下面，我们还是一个例子说明，卷积层前向传播和反向传播的原理。

举例

有这样一个卷积层：
输入: $32 \times 512 \times 14 \times 14$

输出：$32 \times 1024 \times 7 \times 7$

卷积层相关的参数：
kernel = 3
stride = 2
padding = 1
dilition = 1
group = 2

这样，可以计算出网络权重的大小为：weight $1024 \times 256 \times 3 \times 3$

下面大概介绍一下在Caffe中，前向传播中，计算的一个步骤：

（1）给权重分配内存空间$(1024, 256, 3, 3)$，显而易见对于group = 2 的一个卷积来说，每一组的卷积其权重大小为$(512, 256, 3, 3)$

（2）通过im2col变换，将输入的四维数据变成方便与卷积计算的二维矩阵，这样输入数据由$(512, 14,14)$ 变成$(512 * 3 * 3, 7 * 7)$，怎样变换呢？按照卷积的操作以$(14, 14)$为例，卷积核在特征图上每一层滑动，就会覆盖$3 * 3$的一个区域，转换一列$(3 * 3, 1)$，因此对于一个特征图$(14, 14)$来说，将会产生$(3 * 3, 7 * 7)$的一个二维矩阵！对于每一个特征图，在下面进行堆积，最终产生$(512 * 3 * 3, 7 * 7)$

（3）此外，对于$(1024, 256, 3, 3)$权重来说，在C++也可以理解成是一个$(1024, 256 * 3 * 3)$的一个二维矩阵，当然如果考虑group的因素则进一步可以拆分为2个$(512, 256 * 3 * 3)$的矩阵

(4)最终将上面转换得来group个输入二维矩阵和其对应的权值二维矩阵相乘，也就是$(512, 256 * 3 * 3)$ 和 $（256 * 3 * 3， 7 * 7）$相乘，注意这样的乘法有group个

以上就是Caffe中，卷积层前向进行的运算！

通过源码，我们可以了解到输入图像怎么经过im2col转换成方便计算的矩阵的、反向传播过程中怎么经过col2im计算输入图像的梯度的包括权值的梯度、Caffe如何处理dilition的！

Conv_layer头文件

#ifndef CAFFE_CONV_LAYER_HPP_
#define CAFFE_CONV_LAYER_HPP_

#include <vector>

#include "caffe/blob.hpp"
#include "caffe/layer.hpp"
#include "caffe/proto/caffe.pb.h"

#include "caffe/layers/base_conv_layer.hpp"

namespace caffe {
//ConvolutionLayer是BaseConvolutionLayer继承类，关于BaseConvolutionLayer后面再说
template <typename Dtype>
class ConvolutionLayer : public BaseConvolutionLayer<Dtype> {
 public:
    explicit ConvolutionLayer(const LayerParameter& param)
      : BaseConvolutionLayer<Dtype>(param) {}

  virtual inline const char* type() const { return "Convolution"; }

 protected:
  virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top);
  virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top);
  virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
  virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
  virtual inline bool reverse_dimensions() { return false; }
  //计算输出特征图的大小
  virtual void compute_output_shape();
};

}  // namespace caffe

#endif  // CAFFE_CONV_LAYER_HPP_

在头文件中，没有有实质性的东西，只是有一个compute_output_shape函数的声明，接下来我们看看其源文件！

Conv_layer源文件

#include <vector>

#include "caffe/layers/conv_layer.hpp"

namespace caffe {

template <typename Dtype>
void ConvolutionLayer<Dtype>::compute_output_shape() {
//获取卷积核大小的数据（只是卷积核大小，并非权重，后面一样）
  const int* kernel_shape_data = this->kernel_shape_.cpu_data();
  //获取stride
  const int* stride_data = this->stride_.cpu_data();
  //padding
  const int* pad_data = this->pad_.cpu_data();
  //dilation
  const int* dilation_data = this->dilation_.cpu_data();
  //dilation
  this->output_shape_.clear();
  //大循环，对于输入特征图 32 * 512 * 14 * 14，后面两个维度，属于特征图的空间维度
  for (int i = 0; i < this->num_spatial_axes_; ++i) {
    // i + 1 to skip channel axis

    const int input_dim = this->input_shape(i + 1);
    //扩展的卷积核大小，由于有dilation的存在，相当于将卷积核扩大到
    // kernel = dilation * (kernel - 1) + 1
    const int kernel_extent = dilation_data[i] * (kernel_shape_data[i] - 1) + 1;
    //输出特征图大小 = （input_size + 2 * pad - kernel）/ stride + 1
    //向下取整
    const int output_dim = (input_dim + 2 * pad_data[i] - kernel_extent)
        / stride_data[i] + 1;
    this->output_shape_.push_back(output_dim);
  }
}

template <typename Dtype>
void ConvolutionLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
      //获取权重指针
  const Dtype* weight = this->blobs_[0]->cpu_data();
  for (int i = 0; i < bottom.size(); ++i) {//大循环，对于所有输入卷积
    const Dtype* bottom_data = bottom[i]->cpu_data();
    Dtype* top_data = top[i]->mutable_cpu_data();
    for (int n = 0; n < this->num_; ++n) {//第一维度，32不参与卷积
    //forward_cpu_gemm不在Conv_Layer中定义
    //那就在BaseConvLayer中定义
    //参数: 输入， 权重， 输出
      this->forward_cpu_gemm(bottom_data + n * this->bottom_dim_, weight,
          top_data + n * this->top_dim_);
      if (this->bias_term_) {
      //以及计算偏置值
        const Dtype* bias = this->blobs_[1]->cpu_data();
        this->forward_cpu_bias(top_data + n * this->top_dim_, bias);
      }
    }
  }
}

//反向传播
//在卷基层中，反向传播，需要对bottom求梯度，也有对weight求梯度
template <typename Dtype>
void ConvolutionLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
      //获取权重指针，不可修改
  const Dtype* weight = this->blobs_[0]->cpu_data();
  //获取权重指针，可修改
  Dtype* weight_diff = this->blobs_[0]->mutable_cpu_diff();
  for (int i = 0; i < top.size(); ++i) {//大循环，对每一个输出求，都要反向传播
    const Dtype* top_diff = top[i]->cpu_diff();//输出的梯度
    const Dtype* bottom_data = bottom[i]->cpu_data();//输入
    Dtype* bottom_diff = bottom[i]->mutable_cpu_diff();//输入的梯度，可修改
    // Bias gradient, if necessary.首先对偏置值求取梯度
    if (this->bias_term_ && this->param_propagate_down_[1]) {
      Dtype* bias_diff = this->blobs_[1]->mutable_cpu_diff();
      for (int n = 0; n < this->num_; ++n) {//第一维度32
        this->backward_cpu_bias(bias_diff, top_diff + n * this->top_dim_);
      }
    }
    //param_propagate_down_表示权重是否求梯度
    //propagate_down表示bottom是否求梯度
    if (this->param_propagate_down_[0] || propagate_down[i]) {
    //权重求梯度
      for (int n = 0; n < this->num_; ++n) {
        // gradient w.r.t. weight. Note that we will accumulate diffs.
        if (this->param_propagate_down_[0]) {
        //权重求梯度
          this->weight_cpu_gemm(bottom_data + n * this->bottom_dim_,
              top_diff + n * this->top_dim_, weight_diff);
        }
        // gradient w.r.t. bottom data, if necessary.
        if (propagate_down[i]) {
        //bottom求梯度
          this->backward_cpu_gemm(top_diff + n * this->top_dim_, weight,
              bottom_diff + n * this->bottom_dim_);
        }
      }
    }
  }
}

#ifdef CPU_ONLY
STUB_GPU(ConvolutionLayer);
#endif

INSTANTIATE_CLASS(ConvolutionLayer);

}  // namespace caffe

conv_layer在实质上并没有说明，卷积层是怎么做的，只是有这样的一个流程！
说一些与卷积实现无关的题外话，卷积层的反向传播可以看出，我们在训练网络的过程中，bottom和weight是否反向传播是相互独立的，在某些应用中，我们可以固定某一层权值不进行求梯度，但梯度依然也可以向后传播！
通过这样的设置，我们能够更加灵活的fine-tuning!

在conv_layer类的前向和反向运算中，分别调用forward_cpu_gemm， weight_cpu_gemm， backward_cpu_gemm显然是在BaseConvolutionLayer类中

BaseConvolutionLayer头文件

#ifndef CAFFE_BASE_CONVOLUTION_LAYER_HPP_
#define CAFFE_BASE_CONVOLUTION_LAYER_HPP_

#include <vector>

#include "caffe/blob.hpp"
#include "caffe/layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/im2col.hpp"

namespace caffe {

template <typename Dtype>
class BaseConvolutionLayer : public Layer<Dtype> {
 public:
  explicit BaseConvolutionLayer(const LayerParameter& param)
      : Layer<Dtype>(param) {}
  virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top);
  virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top);

  virtual inline int MinBottomBlobs() const { return 1; }
  virtual inline int MinTopBlobs() const { return 1; }
  virtual inline bool EqualNumBottomTopBlobs() const { return true; }

 protected:
 //以下这些函数在conv_layer中被调用
  void forward_cpu_gemm(const Dtype* input, const Dtype* weights,
      Dtype* output, bool skip_im2col = false);
  void forward_cpu_bias(Dtype* output, const Dtype* bias);
  void backward_cpu_gemm(const Dtype* input, const Dtype* weights,
      Dtype* output);
  void weight_cpu_gemm(const Dtype* input, const Dtype* output, Dtype*
      weights);
  void backward_cpu_bias(Dtype* bias, const Dtype* input);

#ifndef CPU_ONLY
  void forward_gpu_gemm(const Dtype* col_input, const Dtype* weights,
      Dtype* output, bool skip_im2col = false);
  void forward_gpu_bias(Dtype* output, const Dtype* bias);
  void backward_gpu_gemm(const Dtype* input, const Dtype* weights,
      Dtype* col_output);
  void weight_gpu_gemm(const Dtype* col_input, const Dtype* output, Dtype*
      weights);
  void backward_gpu_bias(Dtype* bias, const Dtype* input);
#endif

  /// @brief The spatial dimensions of the input.
  //获取输入特征图的大小
  //channel_axis_指定哪一个维度是通道的维度，后面的都是特征图的大小
  inline int input_shape(int i) {
    return (*bottom_shape_)[channel_axis_ + i];
  }
//reverse_dimensions函数是否翻转输入输出
//在卷积层中，为false
//在反卷积层中，为true
//这也进一步说明，卷积反卷积前后向传播完全相反
  virtual bool reverse_dimensions() = 0;
  // Compute height_out_ and width_out_ from other parameters.
  virtual void compute_output_shape() = 0;
//以下参数，可从其命名知道其含义，更多的含义在源文件中注释
  /// @brief The spatial dimensions of a filter kernel.
  Blob<int> kernel_shape_;
  /// @brief The spatial dimensions of the stride.
  Blob<int> stride_;
  /// @brief The spatial dimensions of the padding.
  Blob<int> pad_;
  /// @brief The spatial dimensions of the dilation.
  Blob<int> dilation_;
  /// @brief The spatial dimensions of the convolution input.
  Blob<int> conv_input_shape_;
  /// @brief The spatial dimensions of the col_buffer.
  vector<int> col_buffer_shape_; //im2col的后矩阵的大小
  /// @brief The spatial dimensions of the output.
  vector<int> output_shape_;
  const vector<int>* bottom_shape_;

  int num_spatial_axes_;
  int bottom_dim_;
  int top_dim_;

  int channel_axis_;
  int num_;
  int channels_;
  int group_;
  int out_spatial_dim_;
  int weight_offset_;
  int num_output_;
  bool bias_term_;
  bool is_1x1_;
  bool force_nd_im2col_;

 private:
 //im2col以及col2im
  inline void conv_im2col_cpu(const Dtype* data, Dtype* col_buff) {
    if (!force_nd_im2col_ && num_spatial_axes_ == 2) {//如果特征图是二维的情况
      im2col_cpu(data, conv_in_channels_,
          conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
          kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
          pad_.cpu_data()[0], pad_.cpu_data()[1],
          stride_.cpu_data()[0], stride_.cpu_data()[1],
          dilation_.cpu_data()[0], dilation_.cpu_data()[1], col_buff);
    } else {//如果特征图是多维的情况
      im2col_nd_cpu(data, num_spatial_axes_, conv_input_shape_.cpu_data(),
          col_buffer_shape_.data(), kernel_shape_.cpu_data(),
          pad_.cpu_data(), stride_.cpu_data(), dilation_.cpu_data(), col_buff);
    }
  }
  inline void conv_col2im_cpu(const Dtype* col_buff, Dtype* data) {
    if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
      col2im_cpu(col_buff, conv_in_channels_,
          conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
          kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
          pad_.cpu_data()[0], pad_.cpu_data()[1],
          stride_.cpu_data()[0], stride_.cpu_data()[1],
          dilation_.cpu_data()[0], dilation_.cpu_data()[1], data);
    } else {
      col2im_nd_cpu(col_buff, num_spatial_axes_, conv_input_shape_.cpu_data(),
          col_buffer_shape_.data(), kernel_shape_.cpu_data(),
          pad_.cpu_data(), stride_.cpu_data(), dilation_.cpu_data(), data);
    }
  }
#ifndef CPU_ONLY
  inline void conv_im2col_gpu(const Dtype* data, Dtype* col_buff) {
    if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
      im2col_gpu(data, conv_in_channels_,
          conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
          kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
          pad_.cpu_data()[0], pad_.cpu_data()[1],
          stride_.cpu_data()[0], stride_.cpu_data()[1],
          dilation_.cpu_data()[0], dilation_.cpu_data()[1], col_buff);
    } else {
      im2col_nd_gpu(data, num_spatial_axes_, num_kernels_im2col_,
          conv_input_shape_.gpu_data(), col_buffer_.gpu_shape(),
          kernel_shape_.gpu_data(), pad_.gpu_data(),
          stride_.gpu_data(), dilation_.gpu_data(), col_buff);
    }
  }
  inline void conv_col2im_gpu(const Dtype* col_buff, Dtype* data) {
    if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
      col2im_gpu(col_buff, conv_in_channels_,
          conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
          kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
          pad_.cpu_data()[0], pad_.cpu_data()[1],
          stride_.cpu_data()[0], stride_.cpu_data()[1],
          dilation_.cpu_data()[0], dilation_.cpu_data()[1], data);
    } else {
      col2im_nd_gpu(col_buff, num_spatial_axes_, num_kernels_col2im_,
          conv_input_shape_.gpu_data(), col_buffer_.gpu_shape(),
          kernel_shape_.gpu_data(), pad_.gpu_data(), stride_.gpu_data(),
          dilation_.gpu_data(), data);
    }
  }
#endif

  int num_kernels_im2col_;
  int num_kernels_col2im_;
  int conv_out_channels_;
  int conv_in_channels_;
  int conv_out_spatial_dim_;
  int kernel_dim_;
  int col_offset_;
  int output_offset_;

  Blob<Dtype> col_buffer_;
  Blob<Dtype> bias_multiplier_;
};

}  // namespace caffe

#endif  // CAFFE_BASE_CONVOLUTION_LAYER_HPP_

BaseConvolutionLayer源文件

#include <algorithm>
#include <vector>

#include "caffe/filler.hpp"
#include "caffe/layers/base_conv_layer.hpp"
#include "caffe/util/im2col.hpp"
#include "caffe/util/math_functions.hpp"

namespace caffe {

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  // Configure the kernel size, padding, stride, and inputs.
  ConvolutionParameter conv_param = this->layer_param_.convolution_param();
  force_nd_im2col_ = conv_param.force_nd_im2col();
  //channel_axis_通道位于的维度上，1
  channel_axis_ = bottom[0]->CanonicalAxisIndex(conv_param.axis());
  //通道之后的维度都认为是特征图的空间维度，2
  const int first_spatial_axis = channel_axis_ + 1;
  //输入的数据的维度,4
  const int num_axes = bottom[0]->num_axes();
  //特征图的空间有多少维度，2
  num_spatial_axes_ = num_axes - first_spatial_axis;
  CHECK_GE(num_spatial_axes_, 0);
  //卷积核的维度至少是以为的
  vector<int> spatial_dim_blob_shape(1, std::max(num_spatial_axes_, 1));
  // Setup filter kernel dimensions (kernel_shape_).
  //kernel_shape_卷积核形状信息，卷积核可以是多维的，也就是可以实现三维卷积
  kernel_shape_.Reshape(spatial_dim_blob_shape);
  int* kernel_shape_data = kernel_shape_.mutable_cpu_data();
  //如果在prototxt中定义了has_kernel_w，则强制为2维卷积
  //后面的几个if语句都是为了处理，二维卷积和多维卷积在prototxt定义的一些冲突
  //后面stride_ padding等都和kernel的处理方式一样
  //这样做的目的是，可以兼容多维卷积
  if (conv_param.has_kernel_h() || conv_param.has_kernel_w()) {
    CHECK_EQ(num_spatial_axes_, 2)
        << "kernel_h & kernel_w can only be used for 2D convolution.";
    CHECK_EQ(0, conv_param.kernel_size_size())
        << "Either kernel_size or kernel_h/w should be specified; not both.";
    kernel_shape_data[0] = conv_param.kernel_h();
    kernel_shape_data[1] = conv_param.kernel_w();
  } else {
    const int num_kernel_dims = conv_param.kernel_size_size();
    CHECK(num_kernel_dims == 1 || num_kernel_dims == num_spatial_axes_)
        << "kernel_size must be specified once, or once per spatial dimension "
        << "(kernel_size specified " << num_kernel_dims << " times; "
        << num_spatial_axes_ << " spatial dims).";
        //循环，获取每一卷积维度上卷积核的大小
      for (int i = 0; i < num_spatial_axes_; ++i) {
        kernel_shape_data[i] =
            conv_param.kernel_size((num_kernel_dims == 1) ? 0 : i);
      }
  }
  for (int i = 0; i < num_spatial_axes_; ++i) {
    CHECK_GT(kernel_shape_data[i], 0) << "Filter dimensions must be nonzero.";
  }
  // Setup stride dimensions (stride_).
  stride_.Reshape(spatial_dim_blob_shape);
  int* stride_data = stride_.mutable_cpu_data();
  if (conv_param.has_stride_h() || conv_param.has_stride_w()) {
    CHECK_EQ(num_spatial_axes_, 2)
        << "stride_h & stride_w can only be used for 2D convolution.";
    CHECK_EQ(0, conv_param.stride_size())
        << "Either stride or stride_h/w should be specified; not both.";
    stride_data[0] = conv_param.stride_h();
    stride_data[1] = conv_param.stride_w();
  } else {
    const int num_stride_dims = conv_param.stride_size();
    CHECK(num_stride_dims == 0 || num_stride_dims == 1 ||
          num_stride_dims == num_spatial_axes_)
        << "stride must be specified once, or once per spatial dimension "
        << "(stride specified " << num_stride_dims << " times; "
        << num_spatial_axes_ << " spatial dims).";
    const int kDefaultStride = 1;
    for (int i = 0; i < num_spatial_axes_; ++i) {
      stride_data[i] = (num_stride_dims == 0) ? kDefaultStride :
          conv_param.stride((num_stride_dims == 1) ? 0 : i);
      CHECK_GT(stride_data[i], 0) << "Stride dimensions must be nonzero.";
    }
  }
  // Setup pad dimensions (pad_).
  pad_.Reshape(spatial_dim_blob_shape);
  int* pad_data = pad_.mutable_cpu_data();
  if (conv_param.has_pad_h() || conv_param.has_pad_w()) {
    CHECK_EQ(num_spatial_axes_, 2)
        << "pad_h & pad_w can only be used for 2D convolution.";
    CHECK_EQ(0, conv_param.pad_size())
        << "Either pad or pad_h/w should be specified; not both.";
    pad_data[0] = conv_param.pad_h();
    pad_data[1] = conv_param.pad_w();
  } else {
    const int num_pad_dims = conv_param.pad_size();
    CHECK(num_pad_dims == 0 || num_pad_dims == 1 ||
          num_pad_dims == num_spatial_axes_)
        << "pad must be specified once, or once per spatial dimension "
        << "(pad specified " << num_pad_dims << " times; "
        << num_spatial_axes_ << " spatial dims).";
    const int kDefaultPad = 0;
    for (int i = 0; i < num_spatial_axes_; ++i) {
      pad_data[i] = (num_pad_dims == 0) ? kDefaultPad :
          conv_param.pad((num_pad_dims == 1) ? 0 : i);
    }
  }
  // Setup dilation dimensions (dilation_).
  dilation_.Reshape(spatial_dim_blob_shape);
  int* dilation_data = dilation_.mutable_cpu_data();
  const int num_dilation_dims = conv_param.dilation_size();
  CHECK(num_dilation_dims == 0 || num_dilation_dims == 1 ||
        num_dilation_dims == num_spatial_axes_)
      << "dilation must be specified once, or once per spatial dimension "
      << "(dilation specified " << num_dilation_dims << " times; "
      << num_spatial_axes_ << " spatial dims).";
  const int kDefaultDilation = 1;
  for (int i = 0; i < num_spatial_axes_; ++i) {
    dilation_data[i] = (num_dilation_dims == 0) ? kDefaultDilation :
                       conv_param.dilation((num_dilation_dims == 1) ? 0 : i);
  }
  // Special case: im2col is the identity for 1x1 convolution with stride 1
  // and no padding, so flag for skipping the buffer and transformation.
  //确定是否为1 * 1的卷积
  is_1x1_ = true;
  for (int i = 0; i < num_spatial_axes_; ++i) {
    is_1x1_ &=
        kernel_shape_data[i] == 1 && stride_data[i] == 1 && pad_data[i] == 0;
    if (!is_1x1_) { break; }
  }
  // Configure output channels and groups.
  //输入通道数 512
  channels_ = bottom[0]->shape(channel_axis_);
  //输出通道数 1024
  num_output_ = this->layer_param_.convolution_param().num_output();
  CHECK_GT(num_output_, 0);
  // group_ = 2
  group_ = this->layer_param_.convolution_param().group();
 //分组卷积当然需要被group_整除
  CHECK_EQ(channels_ % group_, 0);
  CHECK_EQ(num_output_ % group_, 0)
      << "Number of output should be multiples of group.";
  if (reverse_dimensions()) {//如果输入和输出反向的话，则
    conv_out_channels_ = channels_; // 512
    conv_in_channels_ = num_output_;// 1024
  } else {
    conv_out_channels_ = num_output_;// 1024
    conv_in_channels_ = channels_;//512
  }
  // Handle the parameters: weights and biases.
  // - blobs_[0] holds the filter weights
  // - blobs_[1] holds the biases (optional)
  //在理论上，权值应该是一个 1024 * 256 * 3 * 3的一个四维的矩阵
  vector<int> weight_shape(2);
  weight_shape[0] = conv_out_channels_;//第一维度大小为 输出通道数 1024
  weight_shape[1] = conv_in_channels_ / group_; //第二维度为 输入通道/group_ = 256
  for (int i = 0; i < num_spatial_axes_; ++i) {
    weight_shape.push_back(kernel_shape_data[i]);//卷积核，这本例中为3 * 3
  }
  //在Caffe中，每一层的Blob_表示该层一些参量（一般都是可学习的）
  //在卷积层（反卷积层）中，Blob_[0]为权值参量，Blob_[1]为偏置值
  bias_term_ = this->layer_param_.convolution_param().bias_term();
  vector<int> bias_shape(bias_term_, num_output_);
  if (this->blobs_.size() > 0) {
    CHECK_EQ(1 + bias_term_, this->blobs_.size())
        << "Incorrect number of weight blobs.";
    if (weight_shape != this->blobs_[0]->shape()) {
      Blob<Dtype> weight_shaped_blob(weight_shape);
      LOG(FATAL) << "Incorrect weight shape: expected shape "
          << weight_shaped_blob.shape_string() << "; instead, shape was "
          << this->blobs_[0]->shape_string();
    }
    if (bias_term_ && bias_shape != this->blobs_[1]->shape()) {
      Blob<Dtype> bias_shaped_blob(bias_shape);
      LOG(FATAL) << "Incorrect bias shape: expected shape "
          << bias_shaped_blob.shape_string() << "; instead, shape was "
          << this->blobs_[1]->shape_string();
    }
    LOG(INFO) << "Skipping parameter initialization";
  } else {
    if (bias_term_) {
      this->blobs_.resize(2);
    } else {
      this->blobs_.resize(1);
    }
    // Initialize and fill the weights:
    // output channels x input channels per-group x kernel height x kernel width
    //为权值分配内存
    this->blobs_[0].reset(new Blob<Dtype>(weight_shape));
    //权值初始化
    shared_ptr<Filler<Dtype> > weight_filler(GetFiller<Dtype>(
        this->layer_param_.convolution_param().weight_filler()));
    weight_filler->Fill(this->blobs_[0].get());
    // If necessary, initialize and fill the biases.
    if (bias_term_) {
      this->blobs_[1].reset(new Blob<Dtype>(bias_shape));
      shared_ptr<Filler<Dtype> > bias_filler(GetFiller<Dtype>(
          this->layer_param_.convolution_param().bias_filler()));
      bias_filler->Fill(this->blobs_[1].get());
    }
  }
  //kernel_dim为对于一个输出特征图来说，其权值指针的偏移量
  //256 * 3 * 3
  kernel_dim_ = this->blobs_[0]->count(1);
  //weight_offset_则是对于每一个group_权值指针偏移
  // 512 * 256 * 3 * 3
  weight_offset_ = conv_out_channels_ * kernel_dim_ / group_;
  // Propagate gradients to the parameters (as directed by backward pass).
  this->param_propagate_down_.resize(this->blobs_.size(), true);
}

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  const int first_spatial_axis = channel_axis_ + 1;
  CHECK_EQ(bottom[0]->num_axes(), first_spatial_axis + num_spatial_axes_)
      << "bottom num_axes may not change.";
  num_ = bottom[0]->count(0, channel_axis_);
  CHECK_EQ(bottom[0]->shape(channel_axis_), channels_)
      << "Input size incompatible with convolution kernel.";
  // TODO: generalize to handle inputs of different shapes.
  for (int bottom_id = 1; bottom_id < bottom.size(); ++bottom_id) {
    CHECK(bottom[0]->shape() == bottom[bottom_id]->shape())
        << "shape mismatch - bottom[0]: " << bottom[0]->shape_string()
        << " vs. bottom[" << bottom_id << "]: "
        << bottom[bottom_id]->shape_string();
  }
  // Shape the tops.
  bottom_shape_ = &bottom[0]->shape();
  //计算输出特征图维度的大小
  compute_output_shape();
  vector<int> top_shape(bottom[0]->shape().begin(),
      bottom[0]->shape().begin() + channel_axis_);
  top_shape.push_back(num_output_);
  for (int i = 0; i < num_spatial_axes_; ++i) {
    top_shape.push_back(output_shape_[i]);//并返回到top_shape
  }
  //为输出分配内存空间
  for (int top_id = 0; top_id < top.size(); ++top_id) {
    top[top_id]->Reshape(top_shape);
  }
  //conv_out_spatial_dim_为输出特征图的空间大小 7 * 7 = 49
  if (reverse_dimensions()) {
    conv_out_spatial_dim_ = bottom[0]->count(first_spatial_axis);
  } else {
    conv_out_spatial_dim_ = top[0]->count(first_spatial_axis);
  }
  //在每一个group中，col_buf的偏移量为256 * 3 * 3 * 7 * 7
  col_offset_ = kernel_dim_ * conv_out_spatial_dim_;
  //每一个group中，输出的偏移量：512 * 7 * 7
  output_offset_ = conv_out_channels_ * conv_out_spatial_dim_ / group_;
  // Setup input dimensions (conv_input_shape_).
  vector<int> bottom_dim_blob_shape(1, num_spatial_axes_ + 1);
  conv_input_shape_.Reshape(bottom_dim_blob_shape);
  int* conv_input_shape_data = conv_input_shape_.mutable_cpu_data();
  //conv_input_shape_data输入特征图的形状（包含通道那个维度）：512 * 14 * 14
  for (int i = 0; i < num_spatial_axes_ + 1; ++i) {
    if (reverse_dimensions()) {
      conv_input_shape_data[i] = top[0]->shape(channel_axis_ + i);
    } else {
      conv_input_shape_data[i] = bottom[0]->shape(channel_axis_ + i);
    }
  }
  // The im2col result buffer will only hold one image at a time to avoid
  // overly large memory usage. In the special case of 1x1 convolution
  // it goes lazily unused to save memory.
  //给col_buffer分配内存空间
  col_buffer_shape_.clear();
  //第一维度的大小为：512 * 3 * 3
  //也就是col_buffer二维数据矩阵的行数
  col_buffer_shape_.push_back(kernel_dim_ * group_);
  //后面两个维度，分别为输出特征图的空间尺寸 7 * 7
  //因此，col_buffer经过im2col之后
  //所得矩阵大小为(512 * 3 * 3, 7 * 7)
  for (int i = 0; i < num_spatial_axes_; ++i) {
    if (reverse_dimensions()) {
      col_buffer_shape_.push_back(input_shape(i + 1));
    } else {
      col_buffer_shape_.push_back(output_shape_[i]);
    }
  }
  col_buffer_.Reshape(col_buffer_shape_);
  //bottom_dim_和top_dim_一次卷积输入数据块和输出数据块的大小(也就是num_ = 1的时候)
  //bottom_dim_ : 512 * 14 * 14
  bottom_dim_ = bottom[0]->count(channel_axis_);
  //top_dim_ ： 1024 * 7 * 7
  top_dim_ = top[0]->count(channel_axis_);
  //num_kernels_im2col_: 1024 * 7 * 7
  num_kernels_im2col_ = conv_in_channels_ * conv_out_spatial_dim_;
  //num_kernels_col2im_ : 512 * 14 * 14
  num_kernels_col2im_ = reverse_dimensions() ? top_dim_ : bottom_dim_;
  // Set up the all ones "bias multiplier" for adding biases by BLAS
  out_spatial_dim_ = top[0]->count(first_spatial_axis);
  if (bias_term_) {
    vector<int> bias_multiplier_shape(1, out_spatial_dim_);
    bias_multiplier_.Reshape(bias_multiplier_shape);
    caffe_set(bias_multiplier_.count(), Dtype(1),
        bias_multiplier_.mutable_cpu_data());
  }
}
//forward_cpu_gemm在卷积层中是前向传播
//在反卷积中，为反向传播
template <typename Dtype>
void BaseConvolutionLayer<Dtype>::forward_cpu_gemm(const Dtype* input,
    const Dtype* weights, Dtype* output, bool skip_im2col) {
  const Dtype* col_buff = input;
  if (!is_1x1_) {
    if (!skip_im2col) {
    //将输入的四维矩阵，通过im2col转换成二维矩阵
    //在所举例子中
    //512, 14, 14 --> 512 * 3 * 3, 7 * 7
      conv_im2col_cpu(input, col_buffer_.mutable_cpu_data());
    }
    //获取转换后的指针
    col_buff = col_buffer_.cpu_data();
  }
  for (int g = 0; g < group_; ++g) {//分组卷积（矩阵相乘）
  //矩阵乘法
  //output = weight * col_buff
  //根据caffe_cpu_gemm函数，来看看其相乘的合法性
  //三个矩阵大小通过offset * g来理解，则矩阵大小为*_offset(*: weight, col, output)
  //根据Reshape函数：
  //weigh_offset: 512 * 256 * 3 * 3
  //col_offset: 256 * 3 * 3 * 7 * 7
  //output_offset: 512 * 7 * 7
  //矩阵的所占空间确定了，再来解析矩阵的行和列
  //weight: (conv_out_channels_ /group_, kernel_dim_) = (512, 256 * 3 * 3)
  //col: (kernel_dim_, conv_out_spatial_dim_) = (256 * 3 * 3,  7 * 7)
  //output: (conv_out_channels_ /group_, conv_out_spatial_dim_) = (512, 7 * 7)
    caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, conv_out_channels_ /
        group_, conv_out_spatial_dim_, kernel_dim_,
        (Dtype)1., weights + weight_offset_ * g, col_buff + col_offset_ * g,
        (Dtype)0., output + output_offset_ * g);
  }
}

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::forward_cpu_bias(Dtype* output,
    const Dtype* bias) {
  caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num_output_,
      out_spatial_dim_, 1, (Dtype)1., bias, bias_multiplier_.cpu_data(),
      (Dtype)1., output);
}
//backward_cpu_gemm在卷积层中，为反向传播
//在反卷积层中，为前向传播
template <typename Dtype>
void BaseConvolutionLayer<Dtype>::backward_cpu_gemm(const Dtype* output,
    const Dtype* weights, Dtype* input) {
  Dtype* col_buff = col_buffer_.mutable_cpu_data();
  if (is_1x1_) {
    col_buff = input;
  }
  for (int g = 0; g < group_; ++g) {//每一个group单独卷积
  //和forward_cpu_gemm一样的，我们来对该矩阵进行分析
  //col_buff = weight' * output('表示转置)
  //weigh_offset: 512 * 256 * 3 * 3
  //col_offset: 256 * 3 * 3 * 7 * 7
  //output_offset: 512 * 7 * 7
  //剩下的就和和forward_cpu_gemm一样的一样的
  //之所以，backward_cpu_gemm和forward_cpu_gemm都能够实现反向和前向传播
  //是因为，输入指针的问题
  //以backward_cpu_gemm举例，若输入的指针分别是:top_diff，weight_data, bottom_diff
  //经过后面conv_col2im_cpu的转换，其本质是对bottom求梯度
  //若输入的指针分别是：bottom_data, weight_data, top_data，其本质是前向传播
  //当然，偏移量和矩阵大小在Reshape中的reverse_dimensions函数，是否调换输入输出
  //在卷积层中，reverse_dimensions函数返回false
  //在反卷积层中，reverse_dimensions函数返回true
  //在有些文献中，Deconvolution也称作convolution_transpose
    caffe_cpu_gemm<Dtype>(CblasTrans, CblasNoTrans, kernel_dim_,
        conv_out_spatial_dim_, conv_out_channels_ / group_,
        (Dtype)1., weights + weight_offset_ * g, output + output_offset_ * g,
        (Dtype)0., col_buff + col_offset_ * g);
  }
  //将col_buf返回为四维的矩阵
  if (!is_1x1_) {
    conv_col2im_cpu(col_buff, input);
  }
}
//对权重求梯度
//同样的input和output指针
//卷积运算来说，input: bottom, output: top_diff
//反卷积运算来说，input: top, output: bottom_diff
template <typename Dtype>
void BaseConvolutionLayer<Dtype>::weight_cpu_gemm(const Dtype* input,
    const Dtype* output, Dtype* weights) {
  const Dtype* col_buff = input;
  //和卷积一样的，首先进行im2col
  //将输入由(512, 14, 14)变成(512 * 3 * 3, 7 * 7) 
  if (!is_1x1_) {
    conv_im2col_cpu(input, col_buffer_.mutable_cpu_data());
    col_buff = col_buffer_.cpu_data();
  }
  for (int g = 0; g < group_; ++g) {//分组卷积
  //同样的
  //weight_offset: 512 * 256 * 3 * 3
  //col_offset_: 256 * 3 * 3 *  7 * 7
  //output_offset: 512 * 7 * 7
  //weight_diff = output * col_buf'
  //weight_diff : (512, 256 * 3 * 3)
  //output: (512, 7 * 7)
  //input:(256 * 3 * 3, 7 * 7)
    caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasTrans, conv_out_channels_ / group_,
        kernel_dim_, conv_out_spatial_dim_,
        (Dtype)1., output + output_offset_ * g, col_buff + col_offset_ * g,
        (Dtype)1., weights + weight_offset_ * g);
  }
}
//后面都是GPU以及偏置求导的相关
template <typename Dtype>
void BaseConvolutionLayer<Dtype>::backward_cpu_bias(Dtype* bias,
    const Dtype* input) {
  caffe_cpu_gemv<Dtype>(CblasNoTrans, num_output_, out_spatial_dim_, 1.,
      input, bias_multiplier_.cpu_data(), 1., bias);
}

#ifndef CPU_ONLY

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::forward_gpu_gemm(const Dtype* input,
    const Dtype* weights, Dtype* output, bool skip_im2col) {
  const Dtype* col_buff = input;
  if (!is_1x1_) {
    if (!skip_im2col) {
      conv_im2col_gpu(input, col_buffer_.mutable_gpu_data());
    }
    col_buff = col_buffer_.gpu_data();
  }
  for (int g = 0; g < group_; ++g) {
    caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, conv_out_channels_ /
        group_, conv_out_spatial_dim_, kernel_dim_,
        (Dtype)1., weights + weight_offset_ * g, col_buff + col_offset_ * g,
        (Dtype)0., output + output_offset_ * g);
  }
}

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::forward_gpu_bias(Dtype* output,
    const Dtype* bias) {
  caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num_output_,
      out_spatial_dim_, 1, (Dtype)1., bias, bias_multiplier_.gpu_data(),
      (Dtype)1., output);
}

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::backward_gpu_gemm(const Dtype* output,
    const Dtype* weights, Dtype* input) {
  Dtype* col_buff = col_buffer_.mutable_gpu_data();
  if (is_1x1_) {
    col_buff = input;
  }
  for (int g = 0; g < group_; ++g) {
    caffe_gpu_gemm<Dtype>(CblasTrans, CblasNoTrans, kernel_dim_,
        conv_out_spatial_dim_, conv_out_channels_ / group_,
        (Dtype)1., weights + weight_offset_ * g, output + output_offset_ * g,
        (Dtype)0., col_buff + col_offset_ * g);
  }
  if (!is_1x1_) {
    conv_col2im_gpu(col_buff, input);
  }
}

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::weight_gpu_gemm(const Dtype* input,
    const Dtype* output, Dtype* weights) {
  const Dtype* col_buff = input;
  if (!is_1x1_) {
    conv_im2col_gpu(input, col_buffer_.mutable_gpu_data());
    col_buff = col_buffer_.gpu_data();
  }
  for (int g = 0; g < group_; ++g) {
    caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasTrans, conv_out_channels_ / group_,
        kernel_dim_, conv_out_spatial_dim_,
        (Dtype)1., output + output_offset_ * g, col_buff + col_offset_ * g,
        (Dtype)1., weights + weight_offset_ * g);
  }
}

template <typename Dtype>
void BaseConvolutionLayer<Dtype>::backward_gpu_bias(Dtype* bias,
    const Dtype* input) {
  caffe_gpu_gemv<Dtype>(CblasNoTrans, num_output_, out_spatial_dim_, 1.,
      input, bias_multiplier_.gpu_data(), 1., bias);
}

#endif  // !CPU_ONLY

INSTANTIATE_CLASS(BaseConvolutionLayer);

}  // namespace caffe

im2col col2im

之前说过了im2col是将三维的输入(channel, height, width)特征图通过卷积的对应关系转换成二维矩阵方便矩阵运算。
1. 在卷积层来说，前向传播首先通过im2col转换成col_buf，然后再和权重矩阵相乘，输出结果。其反向传播，则是权值weight_data和top_diff矩阵相乘，转换成col_buf_diff，然后通过col2im进一步转换成bottom_diff；
2. 对于反卷积层来说，由于在反卷积层Reverse_dim函数返回的是True,因此，其权值的矩阵设定和col_buf矩阵的设定都是按照输入输出对调的情况，因此，在前向传播，其本质是卷积层的反向传播，首先通过weight_data和bottom_data相乘得到col_buf的值，然后通过col2im转换成top_data。在反向传播的过程，其本质是卷积层前向传播，通过im2col将top_diff转换为col_buff，然后权值矩阵weight_data和col_buf矩阵相乘得到bottom_diff。、
以上看出其实im2col、col2im其实就是根据卷积的关系从图像到矩阵，从矩阵到图像的过程。其源码如下:

//data_im：输入图像，三维数据(channel, height, width)
//kernel、pad、stride、dilation卷积的基本信息
//再来说说，data_col的形状，根据对应关系以及上文描述的矩阵相乘所满足的信息
//data_col：（channel * kernel_h * kernel_w, output_w * output_h）
template <typename Dtype>
void im2col_cpu(const Dtype* data_im, const int channels,
    const int height, const int width, const int kernel_h, const int kernel_w,
    const int pad_h, const int pad_w,
    const int stride_h, const int stride_w,
    const int dilation_h, const int dilation_w,
    Dtype* data_col) {
//计算输出特征图的长和宽
  const int output_h = (height + 2 * pad_h -
    (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1;
  const int output_w = (width + 2 * pad_w -
    (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1;
  const int channel_size = height * width;
//第一级循环，channel，同时data_im做数据偏移，则以下循环对每一个通道
  for (int channel = channels; channel--; data_im += channel_size) {
    for (int kernel_row = 0; kernel_row < kernel_h; kernel_row++) {//第二级循环，kernel_h
      for (int kernel_col = 0; kernel_col < kernel_w; kernel_col++) {//第三级循环，kernel_w
      //以上三级循环，为col_buf行数
      //显然以下两级循环，求col_buf的每一行
      //每一行的每一个元素代表一张特征图需要卷积点
        int input_row = -pad_h + kernel_row * dilation_h;//卷积点 相对于卷积核的行位置
        for (int output_rows = output_h; output_rows; output_rows--) {
          if (!is_a_ge_zero_and_a_lt_b(input_row, height)) {//如果当前行坐标大于height，则后面的补零
            for (int output_cols = output_w; output_cols; output_cols--) {
              *(data_col++) = 0;
            }
          } else {
            int input_col = -pad_w + kernel_col * dilation_w;//卷积点，相对于卷积核列位置
            for (int output_col = output_w; output_col; output_col--) {
              if (is_a_ge_zero_and_a_lt_b(input_col, width)) {
                *(data_col++) = data_im[input_row * width + input_col];//获取卷积点相对于图像的坐标点
              } else {
                *(data_col++) = 0;
              }
              input_col += stride_w;//进行偏移
            }
          }
          input_row += stride_h;
        }
      }
    }
  }
}