xtensor
Loading...
Searching...
No Matches
xassigner.hpp
1/***************************************************************************
2 * Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht *
3 * Copyright (c) QuantStack *
4 * *
5 * Distributed under the terms of the BSD 3-Clause License. *
6 * *
7 * The full license is in the file LICENSE, distributed with this software. *
8 ****************************************************************************/
9
10#ifndef XTENSOR_ASSIGNER_HPP
11#define XTENSOR_ASSIGNER_HPP
12
13#include "../core/xassign_proxy.hpp"
14#include "../core/xfunction.hpp"
15
16#if defined(XTENSOR_USE_TBB)
17#include <tbb/tbb.h>
18#endif
19
20#if defined(XTENSOR_USE_OPENMP)
21#include <omp.h>
22#endif
23
24namespace xt
25{
26 /*************
27 * Assigners *
28 *************/
29
30 template <class E1, class E2>
32 {
33 public:
34
35 template <bool simd_assign>
36 void linear_assign(E1& e1, const E2& e2) const;
37
38 template <layout_type L>
39 void stepper_assign(E1& e1, const E2& e2) const;
40
41 void strided_loop_assign(E1& e1, const E2& e2) const;
42
43 private:
44
45 void linear_assign_no_simd(E1& e1, const E2& e2) const;
46 };
47
48#if defined(XTENSOR_USE_TBB)
49 template <class E1, class E2>
50 class tbb_assigner
51 {
52 public:
53
54 template <bool simd_assign>
55 void linear_assign(E1& e1, const E2& e2) const;
56
57 template <layout_type L>
58 void stepper_assign(E1& e1, const E2& e2) const;
59
60 void strided_loop_assign(E1& e1, const E2& e2) const;
61
62 private:
63
64 void linear_assign_no_simd(E1& e1, const E2& e2) const;
65 };
66#endif
67
68#if defined(XTENSOR_USE_OPENMP)
69 template <class E1, class E2>
70 class openmp_assigner
71 {
72 public:
73
74 template <bool simd_assign>
75 void linear_assign(E1& e1, const E2& e2) const;
76
77 template <layout_type L>
78 void stepper_assign(E1& e1, const E2& e2) const;
79
80 void strided_loop_assign(E1& e1, const E2& e2) const;
81
82 private:
83
84 void linear_assign_no_simd(E1& e1, const E2& e2) const;
85 };
86#endif
87
88 template <class E1, class E2>
90 {
91 public:
92
93 template <bool simd_assign>
94 void linear_assign(E1& e1, const E2& e2) const;
95
96 template <layout_type L>
97 void stepper_assign(E1& e1, const E2& e2) const;
98
99 void strided_loop_assign(E1& e1, const E2& e2) const;
100 };
101
102 /****************************
103 * tags - assigners mapping *
104 ****************************/
105
106 template <class E1, class E2, class Tag>
108
109 template <class E1, class E2>
111 {
112 using type = noparallel_assigner<E1, E2>;
113 };
114
115#if defined(XTENSOR_USE_TBB)
116 template <class E1, class E2>
117 struct parallel_assigner<E1, E2, tbb_tag>
118 {
119 using type = tbb_assigner<E1, E2>;
120 };
121#endif
122
123#if defined(XTENSOR_USE_OPENMP)
124 template <class E1, class E2>
125 struct parallel_assigner<E1, E2, openmp_tag>
126 {
127 using type = openmp_assigner<E1, E2>;
128 };
129#endif
130
131 template <class E1, class E2>
133 {
134 using type = waterfall_assigner<E1, E2>;
135 };
136
137 template <class E1, class E2, class Tag>
138 using parallel_assigner_t = typename parallel_assigner<E1, E2, Tag>::type;
139
140 /***********
141 * Helpers *
142 ***********/
143
144 template <class FROM, class TO>
146 {
147 using argument_type = std::decay_t<FROM>;
148 using result_type = std::decay_t<TO>;
149
150 static const bool value = xtl::is_arithmetic<result_type>::value
151 && (sizeof(result_type) < sizeof(argument_type)
152 || (xtl::is_integral<result_type>::value
153 && std::is_floating_point<argument_type>::value));
154 };
155
156 template <class FROM, class TO>
158 {
159 using argument_type = std::decay_t<FROM>;
160 using result_type = std::decay_t<TO>;
161
162 static const bool value = xtl::is_signed<argument_type>::value != xtl::is_signed<result_type>::value;
163 };
164
165 template <class FROM, class TO>
167 {
168 using argument_type = std::decay_t<FROM>;
169 using result_type = std::decay_t<TO>;
170
171 static const bool value = is_narrowing_conversion<argument_type, result_type>::value
172 || has_sign_conversion<argument_type, result_type>::value;
173 };
174
175 namespace detail
176 {
193 template <class T1, class T2>
194 struct conditional_promote_to_complex
195 {
196 static constexpr bool cond = xtl::is_gen_complex<T1>::value && !xtl::is_gen_complex<T2>::value;
197 // Alternative: use std::complex<T2> or xcomplex<T2, T2, bool> depending on T1
198 using type = std::conditional_t<cond, T1, T2>;
199 };
200
201 template <class T1, class T2>
202 using conditional_promote_to_complex_t = typename conditional_promote_to_complex<T1, T2>::type;
203 }
204
205 template <class E1, class E2>
207 {
208 using e1_value_type = typename E1::value_type;
209 using e2_value_type = typename E2::value_type;
210
211 using e2_requested_value_type = std::
212 conditional_t<std::same_as<e2_value_type, bool>, typename E2::bool_load_type, e2_value_type>;
213 using type = detail::conditional_promote_to_complex_t<e1_value_type, e2_requested_value_type>;
214 };
215
216 template <class E1, class E2>
217 using requested_value_type_t = typename requested_value_type<E1, E2>::type;
218
219 template <class E1, class E2>
221 {
222 public:
223
224 using lhs_align_mode = xt_simd::container_alignment_t<E1>;
225 static constexpr bool is_aligned = std::is_same<lhs_align_mode, aligned_mode>::value;
226 using rhs_align_mode = std::conditional_t<is_aligned, inner_aligned_mode, unaligned_mode>;
227 using e1_value_type = typename E1::value_type;
228 using e2_value_type = typename E2::value_type;
229 using value_type = requested_value_type_t<E1, E2>;
230 using simd_type = xt_simd::simd_type<value_type>;
231 using size_type = typename E1::size_type;
232 static constexpr size_type simd_size = xt_simd::simd_traits<value_type>::size;
233 static constexpr bool needs_cast = has_assign_conversion<e2_value_type, e1_value_type>::value;
234 static constexpr bool is_convertible = std::is_convertible<e2_value_type, e1_value_type>::value;
235
236 size_type alignment_begin(const E1& e1, size_type size) const;
237 size_type alignment_end(size_type align_begin, size_type size) const;
238
239 void assign_elements(E1& e1, const E2& e2, size_type begin, size_type end) const;
240 void assign_simd(E1& e1, const E2& e2, size_type begin, size_type end) const;
241
242 // Assigns elements [begin, end) through linear iterators, without SIMD
243 void assign_elements_no_simd(E1& e1, const E2& e2, size_type begin, size_type end) const;
244 };
245
246 template <class E1, class E2, layout_type L>
247 class stepper_assigner
248 {
249 public:
250
251 using lhs_iterator = typename E1::stepper;
252 using rhs_iterator = typename E2::const_stepper;
253 using shape_type = typename E1::shape_type;
254 using index_type = xindex_type_t<shape_type>;
255 using size_type = typename lhs_iterator::size_type;
256 using difference_type = typename lhs_iterator::difference_type;
257
258 stepper_assigner(E1& e1, const E2& e2);
259
260 void run();
261
262 void step(size_type i);
263 void step(size_type i, size_type n);
264 void reset(size_type i);
265
266 void to_end(layout_type);
267
268 private:
269
270 E1& m_e1;
271
272 lhs_iterator m_lhs;
273 rhs_iterator m_rhs;
274
275 index_type m_index;
276 };
277
278 namespace strided_assign_detail
279 {
281 {
282 bool can_do_strided_assign;
283 bool is_row_major;
284 std::size_t inner_loop_size;
285 std::size_t outer_loop_size;
286 std::size_t cut;
287 std::size_t dimension;
288 };
289 }
290
291 template <class E1, class E2>
292 class strided_loop_assigner
293 {
294 public:
295
296 using size_type = std::size_t;
297 using index_type = dynamic_shape<std::size_t>;
298 using loop_sizes_t = strided_assign_detail::loop_sizes_t;
299 using lhs_stepper = typename E1::stepper;
300 using rhs_stepper = typename E2::const_stepper;
301 using e1_value_type = typename E1::value_type;
302 using e2_value_type = typename E2::value_type;
303 using value_type = requested_value_type_t<E1, E2>;
304 using simd_type = std::conditional_t<
305 std::is_same<e1_value_type, bool>::value,
306 xt_simd::simd_bool_type<value_type>,
307 xt_simd::simd_type<value_type>>;
308 static constexpr bool needs_cast = has_assign_conversion<e2_value_type, e1_value_type>::value;
309
310 strided_loop_assigner(E1& e1, const E2& e2);
311
312 bool can_do_strided_assign() const noexcept;
313 size_type inner_loop_size() const noexcept;
314 size_type outer_loop_size() const noexcept;
315
316 // Assigns the outer iterations [begin, end).
317 void run(size_type begin, size_type end) const;
318
319 private:
320
321 void nth_idx(size_type n, index_type& index) const;
322 void next_idx(index_type& index) const;
323
324 E1& m_e1;
325 const E2& m_e2;
326 loop_sizes_t m_loop_sizes;
327 index_type m_max_shape;
328 size_type m_step_dim;
329 size_type m_simd_size;
330 size_type m_simd_rest;
331 };
332
333 /**************************************
334 * noparallel_assigner implementation *
335 **************************************/
336
337 template <class E1, class E2>
338 template <bool simd_assign>
339 void noparallel_assigner<E1, E2>::linear_assign(E1& e1, const E2& e2) const
340 {
341 if constexpr (!simd_assign)
342 {
343 linear_assign_no_simd(e1, e2);
344 }
345 else
346 {
347 using assigner_t = linear_assigner<E1, E2>;
348 using size_type = typename assigner_t::size_type;
349
350 assigner_t helper;
351 size_type size = e1.size();
352 auto align_begin = helper.alignment_begin(e1, size);
353 auto align_end = helper.alignment_end(align_begin, size);
354
355 helper.assign_elements(e1, e2, 0u, align_begin);
356 helper.assign_simd(e1, e2, align_begin, align_end);
357 helper.assign_elements(e1, e2, align_end, size);
358 }
359 }
360
361 template <class E1, class E2>
362 template <layout_type L>
363 void noparallel_assigner<E1, E2>::stepper_assign(E1& e1, const E2& e2) const
364 {
365 stepper_assigner<E1, E2, L> helper(e1, e2);
366 helper.run();
367 }
368
369 template <class E1, class E2>
370 void noparallel_assigner<E1, E2>::strided_loop_assign(E1& e1, const E2& e2) const
371 {
372 strided_loop_assigner<E1, E2> helper(e1, e2);
373 if (!helper.can_do_strided_assign())
374 {
375 stepper_assign<default_assignable_layout(E1::static_layout)>(e1, e2);
376 return;
377 }
378
379 helper.run(0u, helper.outer_loop_size());
380 }
381
382 template <class E1, class E2>
383 void noparallel_assigner<E1, E2>::linear_assign_no_simd(E1& e1, const E2& e2) const
384 {
385 linear_assigner<E1, E2>{}.assign_elements_no_simd(e1, e2, 0u, e1.size());
386 }
387
388 /*******************************
389 * tbb_assigner implementation *
390 *******************************/
391
392#if defined(XTENSOR_USE_TBB)
393 template <class E1, class E2>
394 template <bool simd_assign>
395 void tbb_assigner<E1, E2>::linear_assign(E1& e1, const E2& e2) const
396 {
397 if constexpr (!simd_assign)
398 {
399 linear_assign_no_simd(e1, e2);
400 }
401 else
402 {
403 using assigner_t = linear_assigner<E1, E2>;
404 using size_type = typename assigner_t::size_type;
405 constexpr size_type simd_size = assigner_t::simd_size;
406
407 assigner_t helper;
408 size_type size = e1.size();
409 auto align_begin = helper.alignment_begin(e1, size);
410 auto align_end = helper.alignment_end(align_begin, size);
411
412 helper.assign_elements(e1, e2, 0u, align_begin);
413 if (size >= XTENSOR_TBB_THRESHOLD)
414 {
415 tbb::static_partitioner ap;
416 tbb::parallel_for(
417 align_begin,
418 align_end,
419 simd_size,
420 [&helper, &e1, &e2](size_type i)
421 {
422 helper.assign_simd(e1, e2, i, i + simd_size);
423 },
424 ap
425 );
426 }
427 else
428 {
429 helper.assign_simd(e1, e2, align_begin, align_end);
430 }
431 helper.assign_elements(e1, e2, align_end, size);
432 }
433 }
434
435 template <class E1, class E2>
436 template <layout_type L>
437 void tbb_assigner<E1, E2>::stepper_assign(E1& e1, const E2& e2) const
438 {
439 stepper_assigner<E1, E2, L> helper(e1, e2);
440 helper.run();
441 }
442
443 template <class E1, class E2>
444 void tbb_assigner<E1, E2>::strided_loop_assign(E1& e1, const E2& e2) const
445 {
446 using assigner_t = strided_loop_assigner<E1, E2>;
447 using size_type = typename assigner_t::size_type;
448
449 assigner_t helper(e1, e2);
450 if (!helper.can_do_strided_assign())
451 {
452 stepper_assign<default_assignable_layout(E1::static_layout)>(e1, e2);
453 return;
454 }
455
456 size_type outer_loop_size = helper.outer_loop_size();
457 if (outer_loop_size > XTENSOR_TBB_THRESHOLD / helper.inner_loop_size())
458 {
459 tbb::static_partitioner sp;
460 tbb::parallel_for(
461 tbb::blocked_range<size_type>(size_type(0), outer_loop_size),
462 [&helper](const tbb::blocked_range<size_type>& r)
463 {
464 helper.run(r.begin(), r.end());
465 },
466 sp
467 );
468 }
469 else
470 {
471 helper.run(0u, outer_loop_size);
472 }
473 }
474
475 template <class E1, class E2>
476 void tbb_assigner<E1, E2>::linear_assign_no_simd(E1& e1, const E2& e2) const
477 {
478 using assigner_t = linear_assigner<E1, E2>;
479 using size_type = typename assigner_t::size_type;
480
481 assigner_t helper;
482 tbb::static_partitioner sp;
483 tbb::parallel_for(
484 tbb::blocked_range<size_type>(size_type(0), e1.size()),
485 [&helper, &e1, &e2](const tbb::blocked_range<size_type>& r)
486 {
487 helper.assign_elements_no_simd(e1, e2, r.begin(), r.end());
488 },
489 sp
490 );
491 }
492#endif
493
494 /**********************************
495 * openmp_assigner implementation *
496 **********************************/
497
498#if defined(XTENSOR_USE_OPENMP)
499 template <class E1, class E2>
500 template <bool simd_assign>
501 void openmp_assigner<E1, E2>::linear_assign(E1& e1, const E2& e2) const
502 {
503 if constexpr (!simd_assign)
504 {
505 linear_assign_no_simd(e1, e2);
506 }
507 else
508 {
509 using assigner_t = linear_assigner<E1, E2>;
510 using size_type = typename assigner_t::size_type;
511 constexpr size_type simd_size = assigner_t::simd_size;
512
513 assigner_t helper;
514 size_type size = e1.size();
515 auto align_begin = helper.alignment_begin(e1, size);
516 auto align_end = helper.alignment_end(align_begin, size);
517
518 helper.assign_elements(e1, e2, 0u, align_begin);
519 if (size >= size_type(XTENSOR_OPENMP_TRESHOLD))
520 {
521 // Signed loop index for MSVC, which only supports OpenMP 2.0
522#pragma omp parallel for default(none) shared(helper, align_begin, align_end, e1, e2)
523 for (auto i = static_cast<std::ptrdiff_t>(align_begin);
524 i < static_cast<std::ptrdiff_t>(align_end);
525 i += static_cast<std::ptrdiff_t>(simd_size))
526 {
527 size_type ui = static_cast<size_type>(i);
528 helper.assign_simd(e1, e2, ui, ui + simd_size);
529 }
530 }
531 else
532 {
533 helper.assign_simd(e1, e2, align_begin, align_end);
534 }
535 helper.assign_elements(e1, e2, align_end, size);
536 }
537 }
538
539 template <class E1, class E2>
540 template <layout_type L>
541 void openmp_assigner<E1, E2>::stepper_assign(E1& e1, const E2& e2) const
542 {
543 stepper_assigner<E1, E2, L> helper(e1, e2);
544 helper.run();
545 }
546
547 template <class E1, class E2>
548 void openmp_assigner<E1, E2>::strided_loop_assign(E1& e1, const E2& e2) const
549 {
550 using assigner_t = strided_loop_assigner<E1, E2>;
551 using size_type = typename assigner_t::size_type;
552
553 assigner_t helper(e1, e2);
554 if (!helper.can_do_strided_assign())
555 {
556 stepper_assign<default_assignable_layout(E1::static_layout)>(e1, e2);
557 return;
558 }
559
560 size_type outer_loop_size = helper.outer_loop_size();
561 if (outer_loop_size >= XTENSOR_OPENMP_TRESHOLD / helper.inner_loop_size())
562 {
563 // Each thread assigns a contiguous chunk of outer iterations,
564 // as schedule(static) would do
565#pragma omp parallel default(none) shared(helper, outer_loop_size)
566 {
567 size_type nb_threads = static_cast<size_type>(omp_get_num_threads());
568 size_type thread_id = static_cast<size_type>(omp_get_thread_num());
569 size_type chunk = outer_loop_size / nb_threads;
570 size_type rest = outer_loop_size % nb_threads;
571 size_type begin = thread_id * chunk + std::min(thread_id, rest);
572 size_type end = begin + chunk + (thread_id < rest ? 1u : 0u);
573 helper.run(begin, end);
574 }
575 }
576 else
577 {
578 helper.run(0u, outer_loop_size);
579 }
580 }
581
582 template <class E1, class E2>
583 void openmp_assigner<E1, E2>::linear_assign_no_simd(E1& e1, const E2& e2) const
584 {
585 using assigner_t = linear_assigner<E1, E2>;
586 using size_type = typename assigner_t::size_type;
587
588 assigner_t helper;
589 size_type n = e1.size();
590
591 if (n >= size_type(XTENSOR_OPENMP_TRESHOLD))
592 {
593 // Each thread assigns a contiguous chunk, as schedule(static) would do
594#pragma omp parallel default(none) shared(helper, e1, e2, n)
595 {
596 size_type nb_threads = static_cast<size_type>(omp_get_num_threads());
597 size_type thread_id = static_cast<size_type>(omp_get_thread_num());
598 size_type chunk = n / nb_threads;
599 size_type rest = n % nb_threads;
600 size_type begin = thread_id * chunk + std::min(thread_id, rest);
601 size_type end = begin + chunk + (thread_id < rest ? 1u : 0u);
602 helper.assign_elements_no_simd(e1, e2, begin, end);
603 }
604 }
605 else
606 {
607 helper.assign_elements_no_simd(e1, e2, 0u, n);
608 }
609 }
610#endif
611
612 /*************************************
613 * waterfall_assigner implementation *
614 *************************************/
615
616 template <class E1, class E2>
617 template <bool simd_assign>
618 void waterfall_assigner<E1, E2>::linear_assign(E1& e1, const E2& e2) const
619 {
620#if defined(XTENSOR_USE_TBB)
621 tbb_assigner<E1, E2>{}.template linear_assign<simd_assign>(e1, e2);
622#elif defined(XTENSOR_USE_OPENMP)
623 openmp_assigner<E1, E2>{}.template linear_assign<simd_assign>(e1, e2);
624#else
625 noparallel_assigner<E1, E2>{}.template linear_assign<simd_assign>(e1, e2);
626#endif
627 }
628
629 template <class E1, class E2>
630 template <layout_type L>
631 void waterfall_assigner<E1, E2>::stepper_assign(E1& e1, const E2& e2) const
632 {
633 stepper_assigner<E1, E2, L> helper(e1, e2);
634 helper.run();
635 }
636
637 template <class E1, class E2>
638 void waterfall_assigner<E1, E2>::strided_loop_assign(E1& e1, const E2& e2) const
639 {
640#if defined(XTENSOR_USE_TBB)
641 tbb_assigner<E1, E2>{}.strided_loop_assign(e1, e2);
642#elif defined(XTENSOR_USE_OPENMP)
643 openmp_assigner<E1, E2>{}.strided_loop_assign(e1, e2);
644#else
645 noparallel_assigner<E1, E2>{}.strided_loop_assign(e1, e2);
646#endif
647 }
648
649 /**********************************
650 * linear_assigner implementation *
651 **********************************/
652
653 template <class E1, class E2>
654 auto linear_assigner<E1, E2>::alignment_begin(const E1& e1, size_type size) const -> size_type
655 {
656 size_type align_begin = is_aligned ? 0 : xt_simd::get_alignment_offset(e1.data(), size, simd_size);
657 return align_begin;
658 }
659
660 template <class E1, class E2>
661 auto linear_assigner<E1, E2>::alignment_end(size_type align_begin, size_type size) const -> size_type
662 {
663 size_type align_end = align_begin + ((size - align_begin) & ~(simd_size - 1));
664 return align_end;
665 }
666
667 template <class E1, class E2>
668 void linear_assigner<E1, E2>::assign_elements(E1& e1, const E2& e2, size_type begin, size_type end) const
669 {
670 for (size_type i = begin; i < end; ++i)
671 {
672 e1.data_element(i) = conditional_cast<needs_cast, e1_value_type>(e2.data_element(i));
673 }
674 }
675
676 template <class E1, class E2>
677 void linear_assigner<E1, E2>::assign_simd(E1& e1, const E2& e2, size_type begin, size_type end) const
678 {
679 for (size_type i = begin; i < end; i += simd_size)
680 {
681 e1.template store_simd<lhs_align_mode>(i, e2.template load_simd<rhs_align_mode, value_type>(i));
682 }
683 }
684
685 template <class E1, class E2>
686 void
687 linear_assigner<E1, E2>::assign_elements_no_simd(E1& e1, const E2& e2, size_type begin, size_type end) const
688 {
689 // If the types are not compatible, this function is still instantiated but never called.
690 // The loop must not be instantiated in that case to avoid compilation errors.
691 if constexpr (!is_convertible)
692 {
693 XTENSOR_PRECONDITION(false, "Internal error: linear_assigner called with unrelated types.");
694 }
695 else
696 {
697 using difference_type = std::ptrdiff_t;
698 auto src = linear_begin(e2) + static_cast<difference_type>(begin);
699 auto dst = linear_begin(e1) + static_cast<difference_type>(begin);
700 for (size_type i = begin; i < end; ++i)
701 {
702 *dst = static_cast<e1_value_type>(*src);
703 ++src;
704 ++dst;
705 }
706 }
707 }
708
709 /***********************************
710 * stepper_assigner implementation *
711 ***********************************/
712
713 template <class E1, class E2, layout_type L>
714 inline stepper_assigner<E1, E2, L>::stepper_assigner(E1& e1, const E2& e2)
715 : m_e1(e1)
716 , m_lhs(e1.stepper_begin(e1.shape()))
717 , m_rhs(e2.stepper_begin(e1.shape()))
718 , m_index(xtl::make_sequence<index_type>(e1.shape().size(), size_type(0)))
719 {
720 }
721
722 template <class E1, class E2, layout_type L>
723 inline void stepper_assigner<E1, E2, L>::run()
724 {
725 using tmp_size_type = typename E1::size_type;
726 using argument_type = std::decay_t<decltype(*m_rhs)>;
727 using result_type = std::decay_t<decltype(*m_lhs)>;
728 constexpr bool needs_cast = has_assign_conversion<argument_type, result_type>::value;
729
730 tmp_size_type s = m_e1.size();
731 for (tmp_size_type i = 0; i < s; ++i)
732 {
734 stepper_tools<L>::increment_stepper(*this, m_index, m_e1.shape());
735 }
736 }
737
738 template <class E1, class E2, layout_type L>
739 inline void stepper_assigner<E1, E2, L>::step(size_type i)
740 {
741 m_lhs.step(i);
742 m_rhs.step(i);
743 }
744
745 template <class E1, class E2, layout_type L>
746 inline void stepper_assigner<E1, E2, L>::step(size_type i, size_type n)
747 {
748 m_lhs.step(i, n);
749 m_rhs.step(i, n);
750 }
751
752 template <class E1, class E2, layout_type L>
753 inline void stepper_assigner<E1, E2, L>::reset(size_type i)
754 {
755 m_lhs.reset(i);
756 m_rhs.reset(i);
757 }
758
759 template <class E1, class E2, layout_type L>
760 inline void stepper_assigner<E1, E2, L>::to_end(layout_type l)
761 {
762 m_lhs.to_end(l);
763 m_rhs.to_end(l);
764 }
765
766 /****************************************
767 * strided_loop_assigner implementation *
768 ****************************************/
769
770 namespace strided_assign_detail
771 {
772 template <layout_type layout>
773 struct idx_tools;
774
775 template <>
777 {
778 template <class T>
779 static void next_idx(T& outer_index, const T& outer_shape)
780 {
781 auto i = outer_index.size();
782 for (; i > 0; --i)
783 {
784 if (outer_index[i - 1] + 1 >= outer_shape[i - 1])
785 {
786 outer_index[i - 1] = 0;
787 }
788 else
789 {
790 outer_index[i - 1]++;
791 break;
792 }
793 }
794 }
795
796 template <class T>
797 static void nth_idx(size_t n, T& outer_index, const T& outer_shape)
798 {
799 dynamic_shape<std::size_t> stride_sizes;
800 xt::resize_container(stride_sizes, outer_shape.size());
801 // compute strides
802 using size_type = typename T::size_type;
803 for (size_type i = outer_shape.size(); i > 0; i--)
804 {
805 stride_sizes[i - 1] = (i == outer_shape.size()) ? 1 : stride_sizes[i] * outer_shape[i];
806 }
807
808 // compute index
809 for (size_type i = 0; i < outer_shape.size(); i++)
810 {
811 auto d_idx = n / stride_sizes[i];
812 outer_index[i] = d_idx;
813 n -= d_idx * stride_sizes[i];
814 }
815 }
816 };
817
818 template <>
820 {
821 template <class T>
822 static void next_idx(T& outer_index, const T& outer_shape)
823 {
824 using size_type = typename T::size_type;
825 size_type i = 0;
826 auto sz = outer_index.size();
827 for (; i < sz; ++i)
828 {
829 if (outer_index[i] + 1 >= outer_shape[i])
830 {
831 outer_index[i] = 0;
832 }
833 else
834 {
835 outer_index[i]++;
836 break;
837 }
838 }
839 }
840
841 template <class T>
842 static void nth_idx(size_t n, T& outer_index, const T& outer_shape)
843 {
844 dynamic_shape<std::size_t> stride_sizes;
845 xt::resize_container(stride_sizes, outer_shape.size());
846
847 using size_type = typename T::size_type;
848
849 // compute required strides
850 for (size_type i = 0; i < outer_shape.size(); i++)
851 {
852 stride_sizes[i] = (i == 0) ? 1 : stride_sizes[i - 1] * outer_shape[i - 1];
853 }
854
855 // compute index
856 for (size_type i = outer_shape.size(); i > 0;)
857 {
858 i--;
859 auto d_idx = n / stride_sizes[i];
860 outer_index[i] = d_idx;
861 n -= d_idx * stride_sizes[i];
862 }
863 }
864 };
865
866 template <layout_type L, class S>
867 struct check_strides_functor
868 {
869 using strides_type = S;
870
871 check_strides_functor(const S& strides)
872 : m_cut(L == layout_type::row_major ? 0 : strides.size())
873 , m_strides(strides)
874 {
875 }
876
877 template <class T, layout_type LE = L>
878 std::enable_if_t<LE == layout_type::row_major, std::size_t> operator()(const T& el)
879 {
880 // All dimenions less than var have differing strides
881 auto var = check_strides_overlap<layout_type::row_major>::get(m_strides, el.strides());
882 if (var > m_cut)
883 {
884 m_cut = var;
885 }
886 return m_cut;
887 }
888
889 template <class T, layout_type LE = L>
890 std::enable_if_t<LE == layout_type::column_major, std::size_t> operator()(const T& el)
891 {
892 auto var = check_strides_overlap<layout_type::column_major>::get(m_strides, el.strides());
893 // All dimensions >= var have differing strides
894 if (var < m_cut)
895 {
896 m_cut = var;
897 }
898 return m_cut;
899 }
900
901 template <class T>
902 std::size_t operator()(const xt::xscalar<T>& /*el*/)
903 {
904 return m_cut;
905 }
906
907 template <class F, class... CT>
908 std::size_t operator()(const xt::xfunction<F, CT...>& xf)
909 {
910 xt::for_each(*this, xf.arguments());
911 return m_cut;
912 }
913
914 private:
915
916 std::size_t m_cut;
917 const strides_type& m_strides;
918 };
919
920 template <bool possible = true, class E1, class E2, std::enable_if_t<!strided_expression<E1> || !possible, bool> = true>
921 loop_sizes_t get_loop_sizes(const E1& e1, const E2&)
922 {
923 return {false, true, 1, e1.size(), e1.dimension(), e1.dimension()};
924 }
925
926 template <bool possible = true, class E1, class E2, std::enable_if_t<strided_expression<E1> && possible, bool> = true>
927 loop_sizes_t get_loop_sizes(const E1& e1, const E2& e2)
928 {
929 using shape_value_type = typename E1::shape_type::value_type;
930 bool is_row_major = true;
931
932 // Try to find a row-major scheme first, where the outer loop is on the first N = `cut`
933 // dimensions, and the inner loop is
934 is_row_major = true;
935 auto is_zero = [](auto i)
936 {
937 return i == 0;
938 };
939 auto&& strides = e1.strides();
940 auto it_bwd = std::find_if_not(strides.rbegin(), strides.rend(), is_zero);
941 bool de1_row_contiguous = it_bwd != strides.rend() && *it_bwd == 1;
942 auto it_fwd = std::find_if_not(strides.begin(), strides.end(), is_zero);
943 bool de1_col_contiguous = it_fwd != strides.end() && *it_fwd == 1;
944 if (de1_row_contiguous)
945 {
946 is_row_major = true;
947 }
948 else if (de1_col_contiguous)
949 {
950 is_row_major = false;
951 }
952 else
953 {
954 // No strided loop possible.
955 return {false, true, 1, e1.size(), e1.dimension(), e1.dimension()};
956 }
957
958 // Cut is the number of dimensions in the outer loop
959 std::size_t cut = 0;
960
961 if (is_row_major)
962 {
963 auto csf = check_strides_functor<layout_type::row_major, decltype(e1.strides())>(e1.strides());
964 cut = csf(e2);
965 // This makes that only one dimension will be treated in the inner loop.
966 if (cut < e1.strides().size() - 1)
967 {
968 // Only make the inner loop go over one dimension by default for now
969 cut = e1.strides().size() - 1;
970 }
971 }
972 else if (!is_row_major)
973 {
974 auto csf = check_strides_functor<layout_type::column_major, decltype(e1.strides())>(e1.strides()
975 );
976 cut = csf(e2);
977 if (cut > 1)
978 {
979 // Only make the inner loop go over one dimension by default for now
980 cut = 1;
981 }
982 } // can't reach here because this would have already triggered the fallback
983
984 std::size_t outer_loop_size = static_cast<std::size_t>(std::accumulate(
985 e1.shape().begin(),
986 e1.shape().begin() + static_cast<std::ptrdiff_t>(cut),
987 shape_value_type(1),
988 std::multiplies<shape_value_type>{}
989 ));
990 std::size_t inner_loop_size = static_cast<std::size_t>(std::accumulate(
991 e1.shape().begin() + static_cast<std::ptrdiff_t>(cut),
992 e1.shape().end(),
993 shape_value_type(1),
994 std::multiplies<shape_value_type>{}
995 ));
996
997 if (!is_row_major)
998 {
999 std::swap(outer_loop_size, inner_loop_size);
1000 }
1001
1002 return {inner_loop_size > 1, is_row_major, inner_loop_size, outer_loop_size, cut, e1.dimension()};
1003 }
1004 }
1005
1006 template <class E1, class E2>
1007 inline strided_loop_assigner<E1, E2>::strided_loop_assigner(E1& e1, const E2& e2)
1008 : m_e1(e1)
1009 , m_e2(e2)
1010 , m_loop_sizes(strided_assign_detail::get_loop_sizes(e1, e2))
1011 , m_max_shape()
1012 , m_step_dim(0)
1013 , m_simd_size(0)
1014 , m_simd_rest(0)
1015 {
1016 if (!m_loop_sizes.can_do_strided_assign)
1017 {
1018 return;
1019 }
1020
1021 auto cut = static_cast<std::ptrdiff_t>(m_loop_sizes.cut);
1022 if (m_loop_sizes.is_row_major)
1023 {
1024 m_max_shape.assign(e1.shape().begin(), e1.shape().begin() + cut);
1025 }
1026 else
1027 {
1028 m_max_shape.assign(e1.shape().begin() + cut, e1.shape().end());
1029 m_step_dim = m_loop_sizes.cut;
1030 }
1031
1032 m_simd_size = m_loop_sizes.inner_loop_size / simd_type::size;
1033 m_simd_rest = m_loop_sizes.inner_loop_size % simd_type::size;
1034 }
1035
1036 template <class E1, class E2>
1037 inline bool strided_loop_assigner<E1, E2>::can_do_strided_assign() const noexcept
1038 {
1039 return m_loop_sizes.can_do_strided_assign;
1040 }
1041
1042 template <class E1, class E2>
1043 inline auto strided_loop_assigner<E1, E2>::inner_loop_size() const noexcept -> size_type
1044 {
1045 return m_loop_sizes.inner_loop_size;
1046 }
1047
1048 template <class E1, class E2>
1049 inline auto strided_loop_assigner<E1, E2>::outer_loop_size() const noexcept -> size_type
1050 {
1051 return m_loop_sizes.outer_loop_size;
1052 }
1053
1054 template <class E1, class E2>
1055 inline void strided_loop_assigner<E1, E2>::run(size_type begin, size_type end) const
1056 {
1057 if (begin >= end)
1058 {
1059 return;
1060 }
1061
1062 const size_type step_dim = m_step_dim;
1063 const size_type simd_size = m_simd_size;
1064 const size_type simd_rest = m_simd_rest;
1065
1066 lhs_stepper lhs = m_e1.stepper_begin(m_e1.shape());
1067 rhs_stepper rhs = m_e2.stepper_begin(m_e1.shape());
1068 index_type index = xtl::make_sequence<index_type>(m_max_shape.size(), size_type(0));
1069
1070 if (begin != size_type(0))
1071 {
1072 nth_idx(begin, index);
1073 for (size_type i = 0; i < index.size(); ++i)
1074 {
1075 lhs.step(i + step_dim, index[i]);
1076 rhs.step(i + step_dim, index[i]);
1077 }
1078 }
1079
1080 for (size_type ox = begin; ox < end; ++ox)
1081 {
1082 for (size_type i = 0; i < simd_size; ++i)
1083 {
1084 lhs.store_simd(rhs.template step_simd<value_type>());
1085 }
1086 for (size_type i = 0; i < simd_rest; ++i)
1087 {
1089 lhs.step_leading();
1090 rhs.step_leading();
1091 }
1092
1093 next_idx(index);
1094
1095 // need to step E1 as well if not contiguous assign (e.g. view)
1096 rhs.to_begin();
1097 if (!E1::contiguous_layout)
1098 {
1099 lhs.to_begin();
1100 for (size_type i = 0; i < index.size(); ++i)
1101 {
1102 lhs.step(i + step_dim, index[i]);
1103 rhs.step(i + step_dim, index[i]);
1104 }
1105 }
1106 else
1107 {
1108 for (size_type i = 0; i < index.size(); ++i)
1109 {
1110 rhs.step(i + step_dim, index[i]);
1111 }
1112 }
1113 }
1114 }
1115
1116 template <class E1, class E2>
1117 inline void strided_loop_assigner<E1, E2>::nth_idx(size_type n, index_type& index) const
1118 {
1119 if (m_loop_sizes.is_row_major)
1120 {
1121 strided_assign_detail::idx_tools<layout_type::row_major>::nth_idx(n, index, m_max_shape);
1122 }
1123 else
1124 {
1125 strided_assign_detail::idx_tools<layout_type::column_major>::nth_idx(n, index, m_max_shape);
1126 }
1127 }
1128
1129 template <class E1, class E2>
1130 inline void strided_loop_assigner<E1, E2>::next_idx(index_type& index) const
1131 {
1132 if (m_loop_sizes.is_row_major)
1133 {
1134 strided_assign_detail::idx_tools<layout_type::row_major>::next_idx(index, m_max_shape);
1135 }
1136 else
1137 {
1138 strided_assign_detail::idx_tools<layout_type::column_major>::next_idx(index, m_max_shape);
1139 }
1140 }
1141}
1142
1143#endif
Multidimensional function operating on xtensor expressions.
auto strides(const E &e, stride_type type=stride_type::normal) noexcept
Get strides of an object.
Definition xstrides.hpp:254
standard mathematical functions for xexpressions
layout_type
Definition xlayout.hpp:24
auto conditional_cast(U &&u)
Perform a type cast when a condition is true.
Definition xutils.hpp:576