Open3D (C++ API)  0.15.1
ContinuousConvBackpropFilter.h
Go to the documentation of this file.
1 // ----------------------------------------------------------------------------
2 // - Open3D: www.open3d.org -
3 // ----------------------------------------------------------------------------
4 // The MIT License (MIT)
5 //
6 // Copyright (c) 2018-2021 www.open3d.org
7 //
8 // Permission is hereby granted, free of charge, to any person obtaining a copy
9 // of this software and associated documentation files (the "Software"), to deal
10 // in the Software without restriction, including without limitation the rights
11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 // copies of the Software, and to permit persons to whom the Software is
13 // furnished to do so, subject to the following conditions:
14 //
15 // The above copyright notice and this permission notice shall be included in
16 // all copies or substantial portions of the Software.
17 //
18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 // IN THE SOFTWARE.
25 // ----------------------------------------------------------------------------
26 
27 #pragma once
28 
29 #include <tbb/parallel_for.h>
30 
31 #include <mutex>
32 
34 
35 namespace open3d {
36 namespace ml {
37 namespace impl {
38 
39 // Implementation of CConvBackropFilterCPU
40 template <class TFeat,
41  class TOut,
42  class TReal,
43  class TIndex,
44  InterpolationMode INTERPOLATION,
45  CoordinateMapping MAPPING,
46  bool ALIGN_CORNERS,
47  bool INDIVIDUAL_EXTENT,
48  bool ISOTROPIC_EXTENT,
49  bool POINT_IMPORTANCE>
50 void _CConvBackropFilterCPU(TOut* filter_backprop,
51  const std::vector<int>& filter_dims,
52  size_t num_out,
53  const TReal* out_positions,
54  size_t num_inp,
55  const TReal* inp_positions,
56  const TFeat* inp_features,
57  const TFeat* inp_importance,
58  size_t neighbors_index_size,
59  const TIndex* neighbors_index,
60  const TFeat* neighbors_importance,
61  const int64_t* neighbors_row_splits,
62  const TReal* extents,
63  const TReal* offsets,
64  const TFeat* out_features_gradient,
65  bool normalize) {
66  const bool NEIGHBORS_IMPORTANCE = neighbors_importance;
67  const int VECSIZE = 32;
68  typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
69  typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
70  InterpolationVec_t interpolation;
71 
72  const int in_channels = filter_dims[filter_dims.size() - 2];
73  const int out_channels = filter_dims[filter_dims.size() - 1];
74 
75  int spatial_filter_size = 1;
76  for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
77  const int total_filter_size =
78  spatial_filter_size * in_channels * out_channels;
79  Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
80  filter_dims[0]);
81 
82  memset(filter_backprop, 0, sizeof(TOut) * total_filter_size);
83  std::mutex filter_backprop_mutex;
84 
85  tbb::parallel_for(
86  tbb::blocked_range<size_t>(0, num_out, 32),
87  [&](const tbb::blocked_range<size_t>& r) {
88  int range_length = r.end() - r.begin();
89 
90  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
91  in_channels * spatial_filter_size, range_length);
92  B.setZero();
93  Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> C(
94  out_channels, range_length);
95 
96  typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
97  Matrix infeat(VECSIZE, in_channels);
98 
99  Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
100  offsets[2]);
101 
102  Eigen::Array<TReal, VECSIZE, 3> inv_extents;
103  if (INDIVIDUAL_EXTENT == false) {
104  if (ISOTROPIC_EXTENT) {
105  inv_extents = 1 / extents[0];
106  } else {
107  inv_extents.col(0) = 1 / extents[0];
108  inv_extents.col(1) = 1 / extents[1];
109  inv_extents.col(2) = 1 / extents[2];
110  }
111  }
112 
113  for (size_t out_idx = r.begin(); out_idx != r.end();
114  ++out_idx) {
115  const int out_col = out_idx - r.begin();
116  const size_t neighbor_start = neighbors_row_splits[out_idx];
117  const size_t neighbor_end =
118  neighbors_row_splits[out_idx + 1];
119  TOut normalizer(0);
120 
121  if (INDIVIDUAL_EXTENT) {
122  if (ISOTROPIC_EXTENT) {
123  inv_extents = 1 / extents[out_idx];
124  } else {
125  inv_extents.col(0) = 1 / extents[3 * out_idx + 0];
126  inv_extents.col(1) = 1 / extents[3 * out_idx + 1];
127  inv_extents.col(2) = 1 / extents[3 * out_idx + 2];
128  }
129  }
130 
131  typename InterpolationVec_t::Weight_t interp_weights;
132  typename InterpolationVec_t::Idx_t interp_indices;
133 
134  int vec_valid_count = 0;
135  Vec_t x, y, z;
136 
137  // set to zero to avoid problems with vectors with less than
138  // VECSIZE valid entries
139  x.setZero();
140  y.setZero();
141  z.setZero();
142  for (size_t n = neighbor_start; n < neighbor_end; ++n) {
143  const size_t inp_idx = neighbors_index[n];
144  const int i = vec_valid_count;
145  x(i) = inp_positions[inp_idx * 3 + 0] -
146  out_positions[out_idx * 3 + 0];
147  y(i) = inp_positions[inp_idx * 3 + 1] -
148  out_positions[out_idx * 3 + 1];
149  z(i) = inp_positions[inp_idx * 3 + 2] -
150  out_positions[out_idx * 3 + 2];
151 
152  const TFeat n_importance =
153  (NEIGHBORS_IMPORTANCE ? neighbors_importance[n]
154  : TFeat(1));
155  normalizer += TOut(n_importance);
156 
157  for (int ic = 0; ic < in_channels; ++ic)
158  infeat(i, ic) =
159  inp_features[inp_idx * in_channels + ic];
160 
161  TFeat importance = TFeat(1);
162  if (POINT_IMPORTANCE)
163  importance = inp_importance[inp_idx];
164  if (NEIGHBORS_IMPORTANCE) importance *= n_importance;
165 
166  if (POINT_IMPORTANCE || NEIGHBORS_IMPORTANCE) {
167  for (int ic = 0; ic < in_channels; ++ic)
168  infeat(i, ic) *= importance;
169  }
170 
171  ++vec_valid_count;
172  if (vec_valid_count == VECSIZE) {
173  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
174  x, y, z, filter_size_xyz, inv_extents,
175  offsets_);
176  interpolation.Interpolate(
177  interp_weights, interp_indices, x, y, z,
178  filter_size_xyz, in_channels);
179  for (int k = 0; k < VECSIZE; ++k)
180  for (int j = 0; j < InterpolationVec_t::Size();
181  ++j) {
182  for (int ic = 0; ic < in_channels; ++ic)
183  B(interp_indices(j, k) + ic, out_col) +=
184  TFeat(interp_weights(j, k)) *
185  infeat(k, ic);
186  }
187  vec_valid_count = 0;
188  }
189  }
190  if (vec_valid_count) {
191  ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
192  x, y, z, filter_size_xyz, inv_extents,
193  offsets_);
194  interpolation.Interpolate(interp_weights,
195  interp_indices, x, y, z,
196  filter_size_xyz, in_channels);
197  for (int k = 0; k < vec_valid_count; ++k)
198  for (int j = 0; j < InterpolationVec_t::Size();
199  ++j) {
200  for (int ic = 0; ic < in_channels; ++ic)
201  B(interp_indices(j, k) + ic, out_col) +=
202  TFeat(interp_weights(j, k)) *
203  infeat(k, ic);
204  }
205  }
206 
207  C.col(out_col) = Eigen::Map<
208  const Eigen::Array<TFeat, Eigen::Dynamic, 1>>(
209  out_features_gradient + out_idx * out_channels,
210  out_channels, 1);
211 
212  if (normalize && normalizer != TOut(0))
213  C.col(out_col) /= TFeat(normalizer);
214 
215  } // out_idx
216 
217  Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic> A(
218  out_channels, spatial_filter_size * in_channels);
219 
220  A = (C * B.transpose()).template cast<TOut>();
221 
222  {
223  std::lock_guard<std::mutex> lock(filter_backprop_mutex);
224  int linear_i = 0;
225  for (int j = 0; j < spatial_filter_size * in_channels; ++j)
226  for (int i = 0; i < out_channels; ++i, ++linear_i) {
227  filter_backprop[linear_i] += A(i, j);
228  }
229  }
230  });
231 }
232 
305 template <class TFeat, class TOut, class TReal, class TIndex>
306 void CConvBackpropFilterCPU(TOut* filter_backprop,
307  const std::vector<int>& filter_dims,
308  size_t num_out,
309  const TReal* out_positions,
310  size_t num_inp,
311  const TReal* inp_positions,
312  const TFeat* inp_features,
313  const TFeat* inp_importance,
314  size_t neighbors_index_size,
315  const TIndex* neighbors_index,
316  const TFeat* neighbors_importance,
317  const int64_t* neighbors_row_splits,
318  const TReal* extents,
319  const TReal* offsets,
320  const TFeat* out_features_gradient,
321  InterpolationMode interpolation,
322  CoordinateMapping coordinate_mapping,
323  bool align_corners,
324  bool individual_extent,
325  bool isotropic_extent,
326  bool normalize) {
327  bool has_importance = inp_importance;
328 
329 #define FN_PARAMETERS \
330  filter_backprop, filter_dims, num_out, out_positions, num_inp, \
331  inp_positions, inp_features, inp_importance, neighbors_index_size, \
332  neighbors_index, neighbors_importance, neighbors_row_splits, \
333  extents, offsets, out_features_gradient, normalize
334 
335 #define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
336  INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, HAS_IMPORTANCE) \
337  if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
338  ALIGN_CORNERS == align_corners && \
339  INDIVIDUAL_EXTENT == individual_extent && \
340  ISOTROPIC_EXTENT == isotropic_extent && \
341  HAS_IMPORTANCE == has_importance) \
342  _CConvBackropFilterCPU<TFeat, TOut, TReal, TIndex, INTERPOLATION, \
343  MAPPING, ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
344  ISOTROPIC_EXTENT, HAS_IMPORTANCE>( \
345  FN_PARAMETERS);
346 
347 #define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
348  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
349  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
350  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
351  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
352  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
353  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
354  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
355  CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
356  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
357  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
358  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
359  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
360  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
361  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
362  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
363  CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
364 
365 #define CALL_TEMPLATE3(INTERPOLATION) \
366  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
367  CALL_TEMPLATE2(INTERPOLATION, \
368  CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
369  CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
370 
371 #define CALL_TEMPLATE4 \
372  CALL_TEMPLATE3(InterpolationMode::LINEAR) \
373  CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
374  CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
375 
377 
378 #undef CALL_TEMPLATE
379 #undef CALL_TEMPLATE2
380 #undef CALL_TEMPLATE3
381 #undef CALL_TEMPLATE4
382 
383 #undef FN_PARAMETERS
384 }
385 
386 } // namespace impl
387 } // namespace ml
388 } // namespace open3d
#define CALL_TEMPLATE4
#define VECSIZE
InterpolationMode
Definition: ContinuousConvTypes.h:37
void _CConvBackropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, bool normalize)
Definition: ContinuousConvBackpropFilter.h:50
void CConvBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvBackpropFilter.h:306
CoordinateMapping
Definition: ContinuousConvTypes.h:45
Definition: PinholeCameraIntrinsic.cpp:35
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204