Open3D (C++ API)  0.15.1
ContinuousConvTransposeBackpropFilter.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // The MIT License (MIT)
5 //
6 // Copyright (c) 2018-2021 www.open3d.org
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a copy
9 // of this software and associated documentation files (the "Software"), to deal
10 // in the Software without restriction, including without limitation the rights
11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 // copies of the Software, and to permit persons to whom the Software is
13 // furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 // IN THE SOFTWARE.
25 // ----------------------------------------------------------------------------
26 
27 #pragma once
28 
29 #include <tbb/parallel_for.h>
30 
31 #include <mutex>
32 
34 
35 namespace open3d {
36 namespace ml {
37 namespace impl {
38 
41 template <class TFeat,
42  class TOut,
43  class TReal,
44  class TIndex,
45  InterpolationMode INTERPOLATION,
46  CoordinateMapping MAPPING,
47  bool ALIGN_CORNERS,
48  bool INDIVIDUAL_EXTENT,
49  bool ISOTROPIC_EXTENT,
50  bool NORMALIZE>
51 void _CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
52  const std::vector<int>& filter_dims,
53  size_t num_out,
54  const TReal* out_positions,
55  const TFeat* out_importance,
56  size_t num_inp,
57  const TReal* inp_positions,
58  const TFeat* inp_features,
59  const TFeat* inp_neighbors_importance_sum,
60  const int64_t* inp_neighbors_row_splits,
61  size_t neighbors_index_size,
62  const TIndex* neighbors_index,
63  const TFeat* neighbors_importance,
64  const int64_t* neighbors_row_splits,
65  const TReal* extents,
66  const TReal* offsets,
67  const TFeat* out_features_gradient) {
68  const bool NEIGHBOR_IMPORTANCE = neighbors_importance;
69  const int VECSIZE = 32;
70  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
71  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
72  InterpolationVec_t interpolation;
73 
74  const int in_channels = filter_dims[filter_dims.size() - 2];
75  const int out_channels = filter_dims[filter_dims.size() - 1];
76 
77  int spatial_filter_size = 1;
78  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
79  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
80  filter_dims[0]);
81 
82  memset(filter_backprop, 0,
83  sizeof(TOut) * spatial_filter_size * in_channels * out_channels);
84  std::mutex filter_backprop_mutex;
85 
86  tbb::parallel_for(
87  tbb::blocked_range<size_t>(0, num_out, 32),
88  [&](const tbb::blocked_range<size_t>& r) {
89  int range_length = r.end() - r.begin();
90 
91  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
92  in_channels * spatial_filter_size, range_length);
93  B.setZero();
94  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> C(
95  out_channels, range_length);
96 
97  typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
98  Matrix infeat(VECSIZE, in_channels);
99 
100  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
101  offsets[2]);
102 
103  Eigen::Array<TReal, VECSIZE, 3> inv_extents;
104  if (INDIVIDUAL_EXTENT == false) {
105  if (ISOTROPIC_EXTENT) {
106  inv_extents = 1 / extents[0];
107  } else {
108  inv_extents.col(0) = 1 / extents[0];
109  inv_extents.col(1) = 1 / extents[1];
110  inv_extents.col(2) = 1 / extents[2];
111  }
112  }
113 
114  for (size_t out_idx = r.begin(); out_idx != r.end();
115  ++out_idx) {
116  const int out_col = out_idx - r.begin();
117  const size_t neighbor_start = neighbors_row_splits[out_idx];
118  const size_t neighbor_end =
119  neighbors_row_splits[out_idx + 1];
120 
121  C.col(out_col) = Eigen::Map<
122  const Eigen::Array<TFeat, Eigen::Dynamic, 1>>(
123  out_features_gradient + out_idx * out_channels,
124  out_channels, 1);
125 
126  typename InterpolationVec_t::Weight_t interp_weights;
127  typename InterpolationVec_t::Idx_t interp_indices;
128 
129  int vec_valid_count = 0;
130  Vec_t x, y, z;
131 
132  // set to zero to avoid problems with vectors with less than
133  // VECSIZE valid entries
134  x.setZero();
135  y.setZero();
136  z.setZero();
137  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
138  const size_t inp_idx = neighbors_index[n];
139 
140  const int i = vec_valid_count;
141  x(i) = out_positions[out_idx * 3 + 0] -
142  inp_positions[inp_idx * 3 + 0];
143  y(i) = out_positions[out_idx * 3 + 1] -
144  inp_positions[inp_idx * 3 + 1];
145  z(i) = out_positions[out_idx * 3 + 2] -
146  inp_positions[inp_idx * 3 + 2];
147 
148  if (INDIVIDUAL_EXTENT) {
149  if (ISOTROPIC_EXTENT) {
150  inv_extents.row(i) = 1 / extents[inp_idx];
151  } else {
152  inv_extents(i, 0) =
153  1 / extents[3 * inp_idx + 0];
154  inv_extents(i, 1) =
155  1 / extents[3 * inp_idx + 1];
156  inv_extents(i, 2) =
157  1 / extents[3 * inp_idx + 2];
158  }
159  }
160 
161  TFeat n_importance = NEIGHBOR_IMPORTANCE
162  ? neighbors_importance[n]
163  : TFeat(1);
164  for (int ic = 0; ic < in_channels; ++ic)
165  infeat(i, ic) =
166  inp_features[inp_idx * in_channels + ic] *
167  n_importance;
168 
169  if (NORMALIZE) {
170  TFeat normalizer(1);
171  if (NEIGHBOR_IMPORTANCE) {
172  if (inp_neighbors_importance_sum[inp_idx] !=
173  TFeat(0))
174  normalizer /= inp_neighbors_importance_sum
175  [inp_idx];
176  } else {
177  size_t num_inp_neighbors;
178  const size_t inp_neighbor_start =
179  inp_neighbors_row_splits[inp_idx];
180  const size_t inp_neighbor_end =
181  inp_neighbors_row_splits[inp_idx + 1];
182  num_inp_neighbors =
183  inp_neighbor_end - inp_neighbor_start;
184  if (num_inp_neighbors > 0)
185  normalizer /= TFeat(num_inp_neighbors);
186  }
187  for (int ic = 0; ic < in_channels; ++ic)
188  infeat(i, ic) *= normalizer;
189  }
190 
191  ++vec_valid_count;
192  if (vec_valid_count == VECSIZE ||
193  n + 1 == neighbor_end) {
194  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
195  x, y, z, filter_size_xyz, inv_extents,
196  offsets_);
197  interpolation.Interpolate(
198  interp_weights, interp_indices, x, y, z,
199  filter_size_xyz, in_channels);
200  for (int k = 0; k < vec_valid_count; ++k) {
201  for (int j = 0; j < InterpolationVec_t::Size();
202  ++j) {
203  for (int ic = 0; ic < in_channels; ++ic)
204  B(interp_indices(j, k) + ic, out_col) +=
205  TFeat(interp_weights(j, k)) *
206  infeat(k, ic);
207  }
208  }
209  vec_valid_count = 0;
210  }
211  }
212 
213  } // out_idx
214 
215  if (out_importance) {
216  for (size_t out_idx = r.begin(); out_idx != r.end();
217  ++out_idx) {
218  const int out_col = out_idx - r.begin();
219  C.col(out_col) *= out_importance[out_idx];
220  }
221  }
222 
223  Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic> A(
224  out_channels, spatial_filter_size * in_channels);
225 
226  A = (C * B.transpose()).template cast<TOut>();
227 
228  {
229  std::lock_guard<std::mutex> lock(filter_backprop_mutex);
230  int linear_i = 0;
231  for (int j = 0; j < spatial_filter_size * in_channels; ++j)
232  for (int i = 0; i < out_channels; ++i, ++linear_i) {
233  filter_backprop[linear_i] += A(i, j);
234  }
235  }
236  });
237 }
238 
321 template <class TFeat, class TOut, class TReal, class TIndex>
322 void CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
323  const std::vector<int>& filter_dims,
324  size_t num_out,
325  const TReal* out_positions,
326  const TFeat* out_importance,
327  size_t num_inp,
328  const TReal* inp_positions,
329  const TFeat* inp_features,
330  const TFeat* inp_neighbors_importance_sum,
331  const int64_t* inp_neighbors_row_splits,
332  size_t neighbors_index_size,
333  const TIndex* neighbors_index,
334  const TFeat* neighbors_importance,
335  const int64_t* neighbors_row_splits,
336  const TReal* extents,
337  const TReal* offsets,
338  const TFeat* out_features_gradient,
339  InterpolationMode interpolation,
340  CoordinateMapping coordinate_mapping,
341  bool align_corners,
342  bool individual_extent,
343  bool isotropic_extent,
344  bool normalize) {
345 #define FN_PARAMETERS \
346  filter_backprop, filter_dims, num_out, out_positions, out_importance, \
347  num_inp, inp_positions, inp_features, \
348  inp_neighbors_importance_sum, inp_neighbors_row_splits, \
349  neighbors_index_size, neighbors_index, neighbors_importance, \
350  neighbors_row_splits, extents, offsets, out_features_gradient
351 
352 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
353  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE) \
354  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
355  ALIGN_CORNERS == align_corners && \
356  INDIVIDUAL_EXTENT == individual_extent && \
357  ISOTROPIC_EXTENT == isotropic_extent && NORMALIZE == normalize) \
358  _CConvTransposeBackpropFilterCPU<TFeat, TOut, TReal, TIndex, \
359  INTERPOLATION, MAPPING, \
360  ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
361  ISOTROPIC_EXTENT, NORMALIZE>( \
362  FN_PARAMETERS);
363 
364 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
365  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
366  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
367  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
368  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
369  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
370  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
371  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
372  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
373  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
374  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
375  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
376  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
377  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
378  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
379  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
380  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
381 
382 #define CALL_TEMPLATE3(INTERPOLATION) \
383  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
384  CALL_TEMPLATE2(INTERPOLATION, \
385  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
386  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
387 
388 #define CALL_TEMPLATE4 \
389  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
390  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
391  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
392 
394 
395 #undef CALL_TEMPLATE
396 #undef CALL_TEMPLATE2
397 #undef CALL_TEMPLATE3
398 #undef CALL_TEMPLATE4
399 
400 #undef FN_PARAMETERS
401 }
402 
403 } // namespace impl
404 } // namespace ml
405 } // namespace open3d
#define CALL_TEMPLATE4
#define VECSIZE
void _CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient)
Definition: ContinuousConvTransposeBackpropFilter.h:51
InterpolationMode
Definition: ContinuousConvTypes.h:37
void CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvTransposeBackpropFilter.h:322
CoordinateMapping
Definition: ContinuousConvTypes.h:45
Definition: PinholeCameraIntrinsic.cpp:35
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204