support identity bin file from file content

include/LightGBM/c_api.h

@@ -62,15 +62,6 @@ DllExport int LGBM_CreateDatasetFromFile(const char* filename,
   const DatesetHandle* reference,
   DatesetHandle* out);
 
-/*!
-* \brief load data set from binary file like the command_line LightGBM do
-* \param filename the name of the file
-* \param out a loaded dataset
-* \return 0 when succeed, -1 when failure happens
-*/
-DllExport int LGBM_CreateDatasetFromBinaryFile(const char* filename,
-  DatesetHandle* out);
-
 /*!
 * \brief create a dataset from CSR format
 * \param indptr pointer to row headers

include/LightGBM/dataset.h

@@ -402,6 +402,8 @@ private:
   int label_idx_ = 0;
   /*! \brief store feature names */
   std::vector<std::string> feature_names_;
+  /*! \brief store feature names */
+  static const char* binary_file_token;
 };
 
 }  // namespace LightGBM

include/LightGBM/dataset_loader.h

@@ -49,7 +49,7 @@ private:
   void ExtractFeaturesFromFile(const char* filename, const Parser* parser, const std::vector<data_size_t>& used_data_indices, Dataset* dataset);
 
   /*! \brief Check can load from binary file */
-  bool CheckCanLoadFromBin(const char* filename);
+  std::string CheckCanLoadFromBin(const char* filename);
 
   const IOConfig& io_config_;
   /*! \brief Random generator*/

src/c_api.cpp

@@ -223,15 +223,6 @@ DllExport int LGBM_CreateDatasetFromFile(const char* filename,
   API_END();
 }
 
-DllExport int LGBM_CreateDatasetFromBinaryFile(const char* filename,
-  DatesetHandle* out) {
-  API_BEGIN();
-  OverallConfig config;
-  DatasetLoader loader(config.io_config, nullptr);
-  *out = loader.LoadFromBinFile(filename, 0, 1);
-  API_END();
-}
-
 DllExport int LGBM_CreateDatasetFromMat(const void* data,
   int data_type,
   int32_t nrow,

src/io/dataset.cpp

@@ -14,6 +14,7 @@
 
 namespace LightGBM {
 
+const char* Dataset::binary_file_token = "______LightGBM_Binary_File_Token______\n";
 
 Dataset::Dataset() {
   num_class_ = 1;
@@ -135,7 +136,8 @@ void Dataset::SaveBinaryFile(const char* bin_filename) {
       Log::Fatal("Cannot write binary data to %s ", bin_filename);
     }
     Log::Info("Saving data to binary file %s", bin_filename);
-
+    size_t size_of_token = std::strlen(binary_file_token);
+    fwrite(binary_file_token, sizeof(char), size_of_token, file);
     // get size of header
     size_t size_of_header = sizeof(num_data_) + sizeof(num_class_) + sizeof(num_features_) + sizeof(num_total_features_) 
       + sizeof(size_t) + sizeof(int) * used_feature_map_.size();

src/io/dataset_loader.cpp

@@ -152,8 +152,8 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
   dataset->data_filename_ = filename;
   dataset->num_class_ = io_config_.num_class;
   dataset->metadata_.Init(filename, dataset->num_class_);
-  bool is_loading_from_binfile = CheckCanLoadFromBin(filename);
-  if (!is_loading_from_binfile) {
+  auto bin_filename = CheckCanLoadFromBin(filename);
+  if (bin_filename.size() == 0) {
     if (!io_config_.use_two_round_loading) {
       // read data to memory
       auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, rank, num_machines,&num_global_data, &used_data_indices);
@@ -185,8 +185,6 @@ Dataset* DatasetLoader::LoadFromFile(const char* filename, int rank, int num_mac
     }
   } else {
     // load data from binary file
-    std::string bin_filename(filename);
-    bin_filename.append(".bin");
     dataset.reset(LoadFromBinFile(bin_filename.c_str(), rank, num_machines));
   }
   // check meta data
@@ -209,8 +207,8 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename,
   dataset->data_filename_ = filename;
   dataset->num_class_ = io_config_.num_class;
   dataset->metadata_.Init(filename, dataset->num_class_);
-  bool is_loading_from_binfile = CheckCanLoadFromBin(filename);
-  if (!is_loading_from_binfile) {
+  auto bin_filename = CheckCanLoadFromBin(filename);
+  if (bin_filename.size() == 0) {
     if (!io_config_.use_two_round_loading) {
       // read data in memory
       auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, 0, 1, &num_global_data, &used_data_indices);
@@ -234,8 +232,6 @@ Dataset* DatasetLoader::LoadFromFileAlignWithOtherDataset(const char* filename,
     }
   } else {
     // load data from binary file
-    std::string bin_filename(filename);
-    bin_filename.append(".bin");
     dataset.reset(LoadFromBinFile(bin_filename.c_str(), 0, 1));
   }
   // not need to check validation data
@@ -261,8 +257,18 @@ Dataset* DatasetLoader::LoadFromBinFile(const char* bin_filename, int rank, int
   size_t buffer_size = 16 * 1024 * 1024;
   auto buffer = std::vector<char>(buffer_size);
   
+  // check token
+  size_t size_of_token = std::strlen(Dataset::binary_file_token);
+  size_t read_cnt = fread(buffer.data(), sizeof(char), size_of_token, file);
+  if (read_cnt != size_of_token) {
+    Log::Fatal("Binary file error: token has the wrong size");
+  }
+  if (std::string(buffer.data()) != std::string(Dataset::binary_file_token)) {
+    Log::Fatal("input file is not LightGBM binary file");
+  }
+
   // read size of header
-  size_t read_cnt = fread(buffer.data(), sizeof(size_t), 1, file);
+  read_cnt = fread(buffer.data(), sizeof(size_t), 1, file);
 
   if (read_cnt != 1) {
     Log::Fatal("Binary file error: header has the wrong size");
@@ -849,7 +855,7 @@ void DatasetLoader::ExtractFeaturesFromFile(const char* filename, const Parser*
 }
 
 /*! \brief Check can load from binary file */
-bool DatasetLoader::CheckCanLoadFromBin(const char* filename) {
+std::string DatasetLoader::CheckCanLoadFromBin(const char* filename) {
   std::string bin_filename(filename);
   bin_filename.append(".bin");
 
@@ -860,12 +866,32 @@ bool DatasetLoader::CheckCanLoadFromBin(const char* filename) {
 #else
   file = fopen(bin_filename.c_str(), "rb");
 #endif
+
   if (file == NULL) {
-    return false;
-  } else {
+    bin_filename = std::string(filename);
+#ifdef _MSC_VER
+    fopen_s(&file, bin_filename.c_str(), "rb");
+#else
+    file = fopen(bin_filename.c_str(), "rb");
+#endif
+    if (file == NULL) {
+      Log::Fatal("cannot open data file %s", bin_filename.c_str());
+    }
+  } 
+
+  size_t buffer_size = 256;
+  auto buffer = std::vector<char>(buffer_size);
+  // read size of token
+  size_t size_of_token = std::strlen(Dataset::binary_file_token);
+  size_t read_cnt = fread(buffer.data(), sizeof(char), size_of_token, file);
   fclose(file);
-    return true;
+  if (read_cnt == size_of_token 
+    && std::string(buffer.data()) == std::string(Dataset::binary_file_token)) {
+    return bin_filename;
+  } else {
+    return std::string();
   }
+
 }
 
 }
\ No newline at end of file