Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "22d6d1fd5a1d05444add1efb8f3480f6932a523f"
Unverified Commit 350d56d5 authored by Guolin Ke's avatar Guolin Ke Committed by GitHub
Browse files

prefetch hint (1.3x speed-up) (#2677) 

* add prefetch for dense bin

* prefetch for ordered bin

* Update meta.h

* Update meta.h

* Update dense_bin.hpp
parent f449a78d
Show whitespace changes
Inline Side-by-side
Showing with 103 additions and 283 deletions
include/LightGBM/bin.h
View file @ 350d56d5
......@@ -343,17 +343,18 @@ class Bin {
* ordered_gradients and ordered_hessians are preprocessed, and they are re-ordered by data_indices.
* Ordered_gradients[i] is aligned with data_indices[i]'s gradients (same for ordered_hessians).
* \param data_indices Used data indices in current leaf
* \param num_data Number of used data
* \param start start index in data_indices
* \param end end index in data_indices
* \param ordered_gradients Pointer to gradients, the data_indices[i]-th data's gradient is ordered_gradients[i]
* \param ordered_hessians Pointer to hessians, the data_indices[i]-th data's hessian is ordered_hessians[i]
* \param out Output Result
*/
virtual void ConstructHistogram(
const data_size_t* data_indices, data_size_t num_data,
const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* ordered_gradients, const score_t* ordered_hessians,
HistogramBinEntry* out) const = 0;
virtual void ConstructHistogram(data_size_t num_data,
virtual void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients, const score_t* ordered_hessians,
HistogramBinEntry* out) const = 0;
......@@ -365,14 +366,15 @@ class Bin {
* ordered_gradients and ordered_hessians are preprocessed, and they are re-ordered by data_indices.
* Ordered_gradients[i] is aligned with data_indices[i]'s gradients (same for ordered_hessians).
* \param data_indices Used data indices in current leaf
* \param num_data Number of used data
* \param start start index in data_indices
* \param end end index in data_indices
* \param ordered_gradients Pointer to gradients, the data_indices[i]-th data's gradient is ordered_gradients[i]
* \param out Output Result
*/
virtual void ConstructHistogram(const data_size_t* data_indices, data_size_t num_data,
virtual void ConstructHistogram(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* ordered_gradients, HistogramBinEntry* out) const = 0;
virtual void ConstructHistogram(data_size_t num_data,
virtual void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients, HistogramBinEntry* out) const = 0;
/*!
......
include/LightGBM/meta.h
View file @ 350d56d5
......@@ -12,6 +12,15 @@
#include <utility>
#include <vector>
#if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))) || defined(__INTEL_COMPILER)
#include <xmmintrin.h>
#define PREFETCH_T0(addr) _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0)
#elif defined(__GNUC__)
#define PREFETCH_T0(addr) __builtin_prefetch(reinterpret_cast<const char*>(addr), 0, 3)
#else
#define PREFETCH_T0(addr) do {} while (0)
#endif
namespace LightGBM {
/*! \brief Type of data size, it is better to use signed type*/
......
src/io/dataset.cpp
View file @ 350d56d5
......@@ -881,6 +881,7 @@ void Dataset::ConstructHistograms(const std::vector<int8_t>& is_feature_used,
// if not use ordered bin
feature_groups_[group]->bin_data_->ConstructHistogram(
data_indices,
0,
num_data,
ptr_ordered_grad,
ptr_ordered_hess,
......@@ -910,6 +911,7 @@ void Dataset::ConstructHistograms(const std::vector<int8_t>& is_feature_used,
// if not use ordered bin
feature_groups_[group]->bin_data_->ConstructHistogram(
data_indices,
0,
num_data,
ptr_ordered_grad,
data_ptr);
......@@ -942,6 +944,7 @@ void Dataset::ConstructHistograms(const std::vector<int8_t>& is_feature_used,
if (ref_ordered_bins[group] == nullptr) {
// if not use ordered bin
feature_groups_[group]->bin_data_->ConstructHistogram(
0,
num_data,
ptr_ordered_grad,
ptr_ordered_hess,
......@@ -970,6 +973,7 @@ void Dataset::ConstructHistograms(const std::vector<int8_t>& is_feature_used,
if (ref_ordered_bins[group] == nullptr) {
// if not use ordered bin
feature_groups_[group]->bin_data_->ConstructHistogram(
0,
num_data,
ptr_ordered_grad,
data_ptr);
......
src/io/dense_bin.hpp
View file @ 350d56d5
......@@ -68,33 +68,15 @@ class DenseBin: public Bin {
BinIterator* GetIterator(uint32_t min_bin, uint32_t max_bin, uint32_t default_bin) const override;
void ConstructHistogram(const data_size_t* data_indices, data_size_t num_data,
void ConstructHistogram(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* ordered_gradients, const score_t* ordered_hessians,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const VAL_T bin0 = data_[data_indices[i]];
const VAL_T bin1 = data_[data_indices[i + 1]];
const VAL_T bin2 = data_[data_indices[i + 2]];
const VAL_T bin3 = data_[data_indices[i + 3]];
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
out[bin0].sum_hessians += ordered_hessians[i];
out[bin1].sum_hessians += ordered_hessians[i + 1];
out[bin2].sum_hessians += ordered_hessians[i + 2];
out[bin3].sum_hessians += ordered_hessians[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
}
for (; i < num_data; ++i) {
const data_size_t prefetch_size = 32 / sizeof(VAL_T);
for (data_size_t i = start; i < end; i++) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + data_indices[i + prefetch_size]);
}
const VAL_T bin = data_[data_indices[i]];
out[bin].sum_gradients += ordered_gradients[i];
out[bin].sum_hessians += ordered_hessians[i];
......@@ -102,33 +84,14 @@ class DenseBin: public Bin {
}
}
void ConstructHistogram(data_size_t num_data,
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients, const score_t* ordered_hessians,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const VAL_T bin0 = data_[i];
const VAL_T bin1 = data_[i + 1];
const VAL_T bin2 = data_[i + 2];
const VAL_T bin3 = data_[i + 3];
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
out[bin0].sum_hessians += ordered_hessians[i];
out[bin1].sum_hessians += ordered_hessians[i + 1];
out[bin2].sum_hessians += ordered_hessians[i + 2];
out[bin3].sum_hessians += ordered_hessians[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
}
for (; i < num_data; ++i) {
const data_size_t prefetch_size = 32 / sizeof(VAL_T);
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + i + prefetch_size);
}
const VAL_T bin = data_[i];
out[bin].sum_gradients += ordered_gradients[i];
out[bin].sum_hessians += ordered_hessians[i];
......@@ -136,56 +99,28 @@ class DenseBin: public Bin {
}
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t num_data,
void ConstructHistogram(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* ordered_gradients,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const VAL_T bin0 = data_[data_indices[i]];
const VAL_T bin1 = data_[data_indices[i + 1]];
const VAL_T bin2 = data_[data_indices[i + 2]];
const VAL_T bin3 = data_[data_indices[i + 3]];
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
}
for (; i < num_data; ++i) {
const data_size_t prefetch_size = 32 / sizeof(VAL_T);
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + data_indices[i + prefetch_size]);
}
const VAL_T bin = data_[data_indices[i]];
out[bin].sum_gradients += ordered_gradients[i];
++out[bin].cnt;
}
}
void ConstructHistogram(data_size_t num_data,
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const VAL_T bin0 = data_[i];
const VAL_T bin1 = data_[i + 1];
const VAL_T bin2 = data_[i + 2];
const VAL_T bin3 = data_[i + 3];
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
}
for (; i < num_data; ++i) {
const data_size_t prefetch_size = 32 / sizeof(VAL_T);
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + i + prefetch_size);
}
const VAL_T bin = data_[i];
out[bin].sum_gradients += ordered_gradients[i];
++out[bin].cnt;
......
src/io/dense_nbits_bin.hpp
View file @ 350d56d5
......@@ -73,41 +73,14 @@ class Dense4bitsBin : public Bin {
inline BinIterator* GetIterator(uint32_t min_bin, uint32_t max_bin, uint32_t default_bin) const override;
void ConstructHistogram(const data_size_t* data_indices, data_size_t num_data,
void ConstructHistogram(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* ordered_gradients, const score_t* ordered_hessians,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const data_size_t idx0 = data_indices[i];
const auto bin0 = (data_[idx0 >> 1] >> ((idx0 & 1) << 2)) & 0xf;
const data_size_t idx1 = data_indices[i + 1];
const auto bin1 = (data_[idx1 >> 1] >> ((idx1 & 1) << 2)) & 0xf;
const data_size_t idx2 = data_indices[i + 2];
const auto bin2 = (data_[idx2 >> 1] >> ((idx2 & 1) << 2)) & 0xf;
const data_size_t idx3 = data_indices[i + 3];
const auto bin3 = (data_[idx3 >> 1] >> ((idx3 & 1) << 2)) & 0xf;
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
out[bin0].sum_hessians += ordered_hessians[i];
out[bin1].sum_hessians += ordered_hessians[i + 1];
out[bin2].sum_hessians += ordered_hessians[i + 2];
out[bin3].sum_hessians += ordered_hessians[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
const data_size_t prefetch_size = 32;
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + (data_indices[i + prefetch_size] >> 1));
}
for (; i < num_data; ++i) {
const data_size_t idx = data_indices[i];
const auto bin = (data_[idx >> 1] >> ((idx & 1) << 2)) & 0xf;
out[bin].sum_gradients += ordered_gradients[i];
......@@ -116,33 +89,14 @@ class Dense4bitsBin : public Bin {
}
}
void ConstructHistogram(data_size_t num_data,
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients, const score_t* ordered_hessians,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const auto bin0 = (data_[i >> 1]) & 0xf;
const auto bin1 = (data_[i >> 1] >> 4) & 0xf;
const auto bin2 = (data_[(i >> 1) + 1]) & 0xf;
const auto bin3 = (data_[(i >> 1) + 1] >> 4) & 0xf;
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
out[bin0].sum_hessians += ordered_hessians[i];
out[bin1].sum_hessians += ordered_hessians[i + 1];
out[bin2].sum_hessians += ordered_hessians[i + 2];
out[bin3].sum_hessians += ordered_hessians[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
const data_size_t prefetch_size = 32;
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + ((i + prefetch_size) >> 1));
}
for (; i < num_data; ++i) {
const auto bin = (data_[i >> 1] >> ((i & 1) << 2)) & 0xf;
out[bin].sum_gradients += ordered_gradients[i];
out[bin].sum_hessians += ordered_hessians[i];
......@@ -150,36 +104,14 @@ class Dense4bitsBin : public Bin {
}
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t num_data,
void ConstructHistogram(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* ordered_gradients,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const data_size_t idx0 = data_indices[i];
const auto bin0 = (data_[idx0 >> 1] >> ((idx0 & 1) << 2)) & 0xf;
const data_size_t idx1 = data_indices[i + 1];
const auto bin1 = (data_[idx1 >> 1] >> ((idx1 & 1) << 2)) & 0xf;
const data_size_t idx2 = data_indices[i + 2];
const auto bin2 = (data_[idx2 >> 1] >> ((idx2 & 1) << 2)) & 0xf;
const data_size_t idx3 = data_indices[i + 3];
const auto bin3 = (data_[idx3 >> 1] >> ((idx3 & 1) << 2)) & 0xf;
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
const data_size_t prefetch_size = 32;
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + (data_indices[i + prefetch_size] >> 1));
}
for (; i < num_data; ++i) {
const data_size_t idx = data_indices[i];
const auto bin = (data_[idx >> 1] >> ((idx & 1) << 2)) & 0xf;
out[bin].sum_gradients += ordered_gradients[i];
......@@ -187,28 +119,14 @@ class Dense4bitsBin : public Bin {
}
}
void ConstructHistogram(data_size_t num_data,
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
HistogramBinEntry* out) const override {
const data_size_t rest = num_data & 0x3;
data_size_t i = 0;
for (; i < num_data - rest; i += 4) {
const auto bin0 = (data_[i >> 1]) & 0xf;
const auto bin1 = (data_[i >> 1] >> 4) & 0xf;
const auto bin2 = (data_[(i >> 1) + 1]) & 0xf;
const auto bin3 = (data_[(i >> 1) + 1] >> 4) & 0xf;
out[bin0].sum_gradients += ordered_gradients[i];
out[bin1].sum_gradients += ordered_gradients[i + 1];
out[bin2].sum_gradients += ordered_gradients[i + 2];
out[bin3].sum_gradients += ordered_gradients[i + 3];
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
const data_size_t prefetch_size = 32;
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(data_.data() + ((i + prefetch_size) >> 1));
}
for (; i < num_data; ++i) {
const auto bin = (data_[i >> 1] >> ((i & 1) << 2)) & 0xf;
out[bin].sum_gradients += ordered_gradients[i];
++out[bin].cnt;
......
src/io/ordered_sparse_bin.hpp
View file @ 350d56d5
......@@ -84,46 +84,17 @@ class OrderedSparseBin: public OrderedBin {
void ConstructHistogram(int leaf, const score_t* gradient, const score_t* hessian,
HistogramBinEntry* out) const override {
const data_size_t prefetch_size = 4;
// get current leaf boundary
const data_size_t start = leaf_start_[leaf];
const data_size_t end = start + leaf_cnt_[leaf];
const int rest = (end - start) % 4;
data_size_t i = start;
// use data on current leaf to construct histogram
for (; i < end - rest; i += 4) {
const VAL_T bin0 = ordered_pair_[i].bin;
const VAL_T bin1 = ordered_pair_[i + 1].bin;
const VAL_T bin2 = ordered_pair_[i + 2].bin;
const VAL_T bin3 = ordered_pair_[i + 3].bin;
const auto g0 = gradient[ordered_pair_[i].ridx];
const auto h0 = hessian[ordered_pair_[i].ridx];
const auto g1 = gradient[ordered_pair_[i + 1].ridx];
const auto h1 = hessian[ordered_pair_[i + 1].ridx];
const auto g2 = gradient[ordered_pair_[i + 2].ridx];
const auto h2 = hessian[ordered_pair_[i + 2].ridx];
const auto g3 = gradient[ordered_pair_[i + 3].ridx];
const auto h3 = hessian[ordered_pair_[i + 3].ridx];
out[bin0].sum_gradients += g0;
out[bin1].sum_gradients += g1;
out[bin2].sum_gradients += g2;
out[bin3].sum_gradients += g3;
out[bin0].sum_hessians += h0;
out[bin1].sum_hessians += h1;
out[bin2].sum_hessians += h2;
out[bin3].sum_hessians += h3;
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(ordered_pair_.data() + i + prefetch_size);
PREFETCH_T0(gradient + ordered_pair_[i + prefetch_size].ridx);
PREFETCH_T0(hessian + ordered_pair_[i + prefetch_size].ridx);
}
for (; i < end; ++i) {
const VAL_T bin0 = ordered_pair_[i].bin;
const auto g0 = gradient[ordered_pair_[i].ridx];
const auto h0 = hessian[ordered_pair_[i].ridx];
......@@ -135,34 +106,15 @@ class OrderedSparseBin: public OrderedBin {
void ConstructHistogram(int leaf, const score_t* gradient,
HistogramBinEntry* out) const override {
const data_size_t prefetch_size = 4;
// get current leaf boundary
const data_size_t start = leaf_start_[leaf];
const data_size_t end = start + leaf_cnt_[leaf];
const int rest = (end - start) % 4;
data_size_t i = start;
// use data on current leaf to construct histogram
for (; i < end - rest; i += 4) {
const VAL_T bin0 = ordered_pair_[i].bin;
const VAL_T bin1 = ordered_pair_[i + 1].bin;
const VAL_T bin2 = ordered_pair_[i + 2].bin;
const VAL_T bin3 = ordered_pair_[i + 3].bin;
const auto g0 = gradient[ordered_pair_[i].ridx];
const auto g1 = gradient[ordered_pair_[i + 1].ridx];
const auto g2 = gradient[ordered_pair_[i + 2].ridx];
const auto g3 = gradient[ordered_pair_[i + 3].ridx];
out[bin0].sum_gradients += g0;
out[bin1].sum_gradients += g1;
out[bin2].sum_gradients += g2;
out[bin3].sum_gradients += g3;
++out[bin0].cnt;
++out[bin1].cnt;
++out[bin2].cnt;
++out[bin3].cnt;
for (data_size_t i = start; i < end; ++i) {
if (i + prefetch_size < end) {
PREFETCH_T0(ordered_pair_.data() + i + prefetch_size);
PREFETCH_T0(gradient + ordered_pair_[i + prefetch_size].ridx);
}
for (; i < end; ++i) {
const VAL_T bin0 = ordered_pair_[i].bin;
const auto g0 = gradient[ordered_pair_[i].ridx];
out[bin0].sum_gradients += g0;
......
src/io/sparse_bin.hpp
View file @ 350d56d5
......@@ -102,25 +102,25 @@ class SparseBin: public Bin {
BinIterator* GetIterator(uint32_t min_bin, uint32_t max_bin, uint32_t default_bin) const override;
void ConstructHistogram(const data_size_t*, data_size_t, const score_t*,
void ConstructHistogram(const data_size_t*, data_size_t, data_size_t, const score_t*,
const score_t*, HistogramBinEntry*) const override {
// Will use OrderedSparseBin->ConstructHistogram() instead
Log::Fatal("Using OrderedSparseBin->ConstructHistogram() instead");
}
void ConstructHistogram(data_size_t, const score_t*,
void ConstructHistogram(data_size_t, data_size_t, const score_t*,
const score_t*, HistogramBinEntry*) const override {
// Will use OrderedSparseBin->ConstructHistogram() instead
Log::Fatal("Using OrderedSparseBin->ConstructHistogram() instead");
}
void ConstructHistogram(const data_size_t*, data_size_t, const score_t*,
void ConstructHistogram(const data_size_t*, data_size_t, data_size_t, const score_t*,
HistogramBinEntry*) const override {
// Will use OrderedSparseBin->ConstructHistogram() instead
Log::Fatal("Using OrderedSparseBin->ConstructHistogram() instead");
}
void ConstructHistogram(data_size_t, const score_t*,
void ConstructHistogram(data_size_t, data_size_t, const score_t*,
HistogramBinEntry*) const override {
// Will use OrderedSparseBin->ConstructHistogram() instead
Log::Fatal("Using OrderedSparseBin->ConstructHistogram() instead");
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment