7 #ifndef XGBOOST_DATA_H_ 8 #define XGBOOST_DATA_H_ 10 #include <dmlc/base.h> 11 #include <dmlc/data.h> 12 #include <dmlc/serializer.h> 13 #include <rabit/rabit.h> 48 static constexpr uint64_t kNumField = 11;
55 uint64_t num_nonzero_{0};
124 void Validate(int32_t device)
const;
137 if (label_order_cache_.size() == labels_.
Size()) {
138 return label_order_cache_;
140 label_order_cache_.resize(labels_.
Size());
141 std::iota(label_order_cache_.begin(), label_order_cache_.end(), 0);
144 [&l](
size_t i1,
size_t i2) {
return std::abs(l[i1]) < std::abs(l[i2]);});
146 return label_order_cache_;
154 void LoadBinary(dmlc::Stream* fi);
159 void SaveBinary(dmlc::Stream* fo)
const;
167 void SetInfo(
const char* key,
const void* dptr,
DataType dtype,
size_t num);
177 void SetInfo(
const char* key, std::string
const& interface_str);
180 const void** out_dptr)
const;
182 void SetFeatureInfo(
const char *key,
const char **info,
const bst_ulong size);
183 void GetFeatureInfo(
const char *field, std::vector<std::string>* out_str_vecs)
const;
193 void Extend(
MetaInfo const& that,
bool accumulate_rows);
197 mutable std::vector<size_t> label_order_cache_;
219 return (this->index == other.
index && this->fvalue == other.
fvalue);
234 BatchParam(int32_t device, int32_t max_bin,
size_t gpu_page_size = 0)
235 : gpu_id{device}, max_bin{max_bin}, gpu_page_size{gpu_page_size} {}
264 if (rabit::IsDistributed() && i + 1 >= offset_vec.size()) {
267 size = offset_vec[i + 1] - offset_vec[i];
269 return {data_vec.data() + offset_vec[i],
280 return offset.
Size() == 0 ? 0 : offset.
Size() - 1;
285 return offset.
Size() *
sizeof(size_t) + data.
Size() *
sizeof(
Entry);
293 offset_vec.push_back(0);
302 SparsePage GetTranspose(
int num_columns)
const;
306 #pragma omp parallel for default(none) shared(ncol) schedule(dynamic, 1) 321 void Push(
const dmlc::RowBlock<uint32_t>& batch);
333 template <
typename AdapterBatchT>
334 uint64_t Push(
const AdapterBatchT& batch,
float missing,
int nthread);
360 class EllpackPageImpl;
394 void SetBaseRowId(
size_t row_id);
396 const EllpackPageImpl*
Impl()
const {
return impl_.get(); }
397 EllpackPageImpl*
Impl() {
return impl_.get(); }
400 std::unique_ptr<EllpackPageImpl> impl_;
407 virtual T& operator*() = 0;
408 virtual const T& operator*()
const = 0;
409 virtual void operator++() = 0;
410 virtual bool AtEnd()
const = 0;
420 CHECK(impl_ !=
nullptr);
425 CHECK(impl_ !=
nullptr);
430 CHECK(impl_ !=
nullptr);
435 CHECK(impl_ !=
nullptr);
436 return !impl_->AtEnd();
440 CHECK(impl_ !=
nullptr);
441 return impl_->AtEnd();
445 std::shared_ptr<BatchIteratorImpl<T>> impl_;
472 this->Info().SetInfo(key, dptr, dtype, num);
474 virtual void SetInfo(
const char* key, std::string
const& interface_str) {
475 this->Info().SetInfo(key, interface_str);
478 virtual const MetaInfo& Info()
const = 0;
488 template <
typename T>
489 bool PageExists()
const;
493 virtual bool SingleColBlock()
const = 0;
499 return Info().num_nonzero_ == Info().num_row_ * Info().num_col_;
512 static DMatrix* Load(
const std::string& uri,
515 const std::string& file_format =
"auto",
516 size_t page_size = kPageSize);
530 template <
typename AdapterT>
531 static DMatrix* Create(AdapterT* adapter,
float missing,
int nthread,
532 const std::string& cache_prefix =
"",
533 size_t page_size = kPageSize);
556 DataIterResetCallback *reset,
563 static const size_t kPageSize = 32UL << 20UL;
571 virtual bool EllpackExists()
const = 0;
572 virtual bool SparsePageExists()
const = 0;
577 return GetRowBatches();
581 inline bool DMatrix::PageExists<EllpackPage>()
const {
582 return this->EllpackExists();
586 inline bool DMatrix::PageExists<SparsePage>()
const {
587 return this->SparsePageExists();
592 return GetColumnBatches();
597 return GetSortedColumnBatches();
602 return GetEllpackBatches(param);
609 namespace serializer {
614 strm->Write(data.
index);
619 return strm->Read(&data->
index) && strm->Read(&data->
fvalue);
625 #endif // XGBOOST_DATA_H_
void operator++()
Definition: data.h:419
std::forward_iterator_tag iterator_category
Definition: data.h:416
void * DataIterHandle
handle to a external data iterator
Definition: c_api.h:191
float bst_float
float type, used for storing statistics
Definition: base.h:111
bool IsDense() const
Whether the matrix is dense.
Definition: data.h:498
void Copy(const HostDeviceVector< T > &other)
BatchIterator(BatchIteratorImpl< T > *impl)
Definition: data.h:417
#define XGBOOST_PARALLEL_SORT(X, Y, Z)
Definition: base.h:68
A page stored in ELLPACK format.
Definition: data.h:367
XGB_EXTERN_C typedef void DataIterResetCallback(DataIterHandle handle)
Callback function prototype for reseting external iterator.
const T & operator*() const
Definition: data.h:429
T & operator*()
Definition: data.h:424
SortedCSCPage()
Definition: data.h:356
virtual void SetInfo(const char *key, std::string const &interface_str)
Definition: data.h:474
std::size_t index_type
Definition: span.h:405
bool operator!=(const BatchParam &other) const
Definition: data.h:236
void * DMatrixHandle
handle to DMatrix
Definition: c_api.h:30
bool AtEnd() const
Definition: data.h:439
void SetBaseRowId(size_t row_id)
Set the base row id for this page.
Definition: data.h:298
void SortRows()
Definition: data.h:304
size_t gpu_page_size
Page size for external memory mode.
Definition: data.h:232
BatchSet(BatchIterator< T > begin_iter)
Definition: data.h:451
uint32_t bst_feature_t
Type for data column (feature) index.
Definition: base.h:114
dmlc::omp_uint bst_omp_uint
define unsigned int for openmp loop
Definition: base.h:261
Internal data structured used by XGBoost during training.
Definition: data.h:464
In-memory storage unit of sparse batch, stored in CSR format.
Definition: data.h:245
A device-and-host vector abstraction layer.
static bool Read(Stream *strm, xgboost::Entry *data)
Definition: data.h:618
Parameters for constructing batches.
Definition: data.h:226
span class implementation, based on ISO++20 span<T>. The interface should be the same.
Definition: span.h:126
uint64_t bst_ulong
unsigned long integers
Definition: base.h:109
BatchSet< T > GetBatches(const BatchParam ¶m={})
Gets batches. Use range based for loop over BatchSet to access individual batches.
int gpu_id
The GPU device to use.
Definition: data.h:228
HostDeviceVector< bst_row_t > offset
Definition: data.h:248
XGBOOST_DEVICE Entry(bst_feature_t index, bst_float fvalue)
constructor with index and value
Definition: data.h:213
SparsePage()
constructor
Definition: data.h:274
BatchIterator< T > begin()
Definition: data.h:452
const EllpackPageImpl * Impl() const
Definition: data.h:396
SortedCSCPage(SparsePage page)
Definition: data.h:357
CSCPage()
Definition: data.h:350
entry to to easily hold returning information
Definition: learner.h:35
std::vector< T > & HostVector()
bool operator==(const Entry &other) const
Definition: data.h:218
Inst operator[](size_t i) const
get i-th row from the batch
Definition: data.h:258
bool operator!=(const BatchIterator &rhs) const
Definition: data.h:434
#define XGBOOST_DEVICE
Tag function as usable by device.
Definition: base.h:84
size_t MemCostBytes() const
Definition: data.h:284
namespace of xgboost
Definition: base.h:102
CSCPage(SparsePage page)
Definition: data.h:351
FeatureType
Definition: data.h:38
virtual void SetInfo(const char *key, const void *dptr, DataType dtype, size_t num)
Definition: data.h:470
defines configuration macros of xgboost.
size_t Size() const
Definition: data.h:279
DataType
data type accepted by xgboost interface
Definition: data.h:30
XGB_EXTERN_C typedef int XGDMatrixCallbackNext(DataIterHandle iter)
Callback function prototype for getting next batch of data.
HostDeviceVector< Entry > data
the data of the segments
Definition: data.h:250
Element from a sparse vector.
Definition: data.h:201
DMLC_DECLARE_TRAITS(is_pod, xgboost::Entry, true)
BatchParam(int32_t device, int32_t max_bin, size_t gpu_page_size=0)
Definition: data.h:234
BatchIterator< T > end()
Definition: data.h:453
static void Write(Stream *strm, const xgboost::Entry &data)
Definition: data.h:613
bst_feature_t index
feature index
Definition: data.h:203
bst_float fvalue
feature value
Definition: data.h:205
static bool CmpValue(const Entry &a, const Entry &b)
reversely compare feature values
Definition: data.h:215
void Resize(size_t new_size, T v=T())
void Clear()
clear the page
Definition: data.h:289
int max_bin
Maximum number of bins per feature for histograms.
Definition: data.h:230
EllpackPageImpl * Impl()
Definition: data.h:397