dataset.h

/* MLPACK 0.2
 *
 * Copyright (c) 2008, 2009 Alexander Gray,
 *                          Garry Boyer,
 *                          Ryan Riegel,
 *                          Nikolaos Vasiloglou,
 *                          Dongryeol Lee,
 *                          Chip Mappus, 
 *                          Nishant Mehta,
 *                          Hua Ouyang,
 *                          Parikshit Ram,
 *                          Long Tran,
 *                          Wee Chin Wong
 *
 * Copyright (c) 2008, 2009 Georgia Institute of Technology
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */
#ifndef DATA_DATASET_H
#define DATA_DATASET_H

#include "fastlib/col/col_string.h"
#include "fastlib/la/matrix.h"
#include "fastlib/math/discrete.h"
#include "fastlib/file/textfile.h"

class TextLineReader;
class TextWriter;

class DatasetFeature {
 public:
  enum Type {
      CONTINUOUS,
      INTEGER,
      NOMINAL
  };
  
 private:
  String name_;
  Type type_;
  ArrayList<String> value_names_;
  
  OBJECT_TRAVERSAL(DatasetFeature) {
    OT_OBJ(name_);
    //OT_OBJ(reinterpret_cast<int &>(type_));
    OT_ENUM_EXPERT(type_, int,
      OT_ENUM_VAL(CONTINUOUS)
      OT_ENUM_VAL(INTEGER)
      OT_ENUM_VAL(NOMINAL));
    OT_OBJ(value_names_);
  }

 void InitGeneral(const char *name_in) {
    name_.Copy(name_in);
    value_names_.Init();
 }

 public:
  void InitContinuous(const char *name_in) {
    InitGeneral(name_in);
    type_ = CONTINUOUS;
  }

  void InitInteger(const char *name_in) {
    InitGeneral(name_in);
    type_ = INTEGER;
  }

  void InitNominal(const char *name_in) {
    InitGeneral(name_in);
    type_ = NOMINAL;
  }
  
  void Format(double value, String *result) const;
  
  success_t Parse(const char *str, double *d) const;
  
  const String& name() const {
    return name_;
  }
  
  Type type() const {
    return type_;
  }
  
  const String& value_name(int value) const {
    DEBUG_ASSERT(type_ == NOMINAL);
    return value_names_[value];
  }
  
  index_t n_values() const {
    return value_names_.size();
  }
  
  ArrayList<String>& value_names() {
    return value_names_;
  }
};

class DatasetInfo {
 private:
  String name_;
  ArrayList<DatasetFeature> features_;

  OBJECT_TRAVERSAL(DatasetInfo) {
    OT_OBJ(name_);
    OT_OBJ(features_);
  }

 public:
  ArrayList<DatasetFeature>& features() {
    return features_;
  }

  const DatasetFeature& feature(index_t attrib_num) const {
    return features_[attrib_num];
  }

  index_t n_features() const {
    return features_.size();
  }
  
  const char *name() const {
    return name_;
  }
  
  void set_name(const char *name_in) {
    name_.Destruct();
    name_.Copy(name_in);
  }

  bool is_all_continuous() const;

  void InitContinuous(index_t n_features,
      const char *name_in = "dataset");

  void Init(const char *name_in = "dataset");

  void WriteArffHeader(TextWriter *writer) const;
  
  void WriteCsvHeader(const char *sep, TextWriter *writer) const;

  void WriteMatrix(const Matrix& matrix, const char *sep,
      TextWriter *writer) const;

  success_t InitFromArff(TextLineReader *reader,
      const char *filename = "dataset");
  
  success_t InitFromCsv(TextLineReader *reader,
      const char *filename = "dataset");

  success_t InitFromFile(TextLineReader *reader,
      const char *filename = "dataset");
  success_t ReadMatrix(TextLineReader *reader, Matrix *matrix) const;

  success_t ReadPoint(TextLineReader *reader, double *point,
      bool *is_done) const;

 private:
  char *SkipSpace_(char *s);

  char *SkipNonspace_(char *s);

  void SkipBlanks_(TextLineReader *reader);

};


class Dataset {
 private:
  Matrix matrix_;
  DatasetInfo info_;
  
  OBJECT_TRAVERSAL(Dataset) {
    OT_OBJ(matrix_);
    OT_OBJ(info_);
  }
  
 public:
  const DatasetInfo& info() const {
    return info_;
  }
  
  DatasetInfo& info() {
    return info_;
  }
  
  index_t n_features() const {
    return matrix_.n_rows();
  }
  
  index_t n_points() const {
    return matrix_.n_cols();
  }

  index_t n_labels() const;

  void GetLabels(ArrayList<double> &labels_list,
                 ArrayList<index_t> &labels_index,
                 ArrayList<index_t> &labels_ct,
                 ArrayList<index_t> &labels_startpos) const;
 
  double get(index_t feature, index_t point) const {
    return matrix_.get(feature, point);
  }
  
  int get_int(index_t feature, index_t point) const {
    double d = get(feature, point);
    int i = int(d);
    DEBUG_ASSERT(d == double(i));
    return i;
  }
  
  void set(index_t feature, index_t point, double d) {
    matrix_.set(feature, point, d);
  }
  
  const double *point(index_t point) const {
    return matrix_.GetColumnPtr(point);
  }
  double *point(index_t point) {
    return matrix_.GetColumnPtr(point);
  }
  
  const Matrix& matrix() const {
    return matrix_;
  }
  Matrix& matrix() {
    return matrix_;
  }
  
  void Format(index_t feature, index_t point, String *result) const {
    info_.feature(feature).Format(get(feature, point), result);
  }
  
  void InitBlank() {
  }
  
  success_t InitFromFile(const char *fname);
  
  success_t InitFromFile(TextLineReader *reader,
      const char *filename = "dataset");
  
  success_t WriteCsv(const char *fname, bool header = false) const;

  success_t WriteArff(const char *fname) const;

  void CopyMatrix(const Matrix& matrix_in) {
    InitBlank();
    matrix_.Copy(matrix_in);
    info_.InitContinuous(matrix_.n_rows());
  }
  
  void OwnMatrix(Matrix* matrix_in) {
    InitBlank();
    matrix_.Own(matrix_in);
    info_.InitContinuous(matrix_.n_rows());
  }
  
  void AliasMatrix(const Matrix& matrix_in) {
    InitBlank();
    matrix_.Alias(matrix_in);
    info_.InitContinuous(matrix_.n_rows());
  }
  
  //--- Cross-validation features ---

  void SplitTrainTest(int folds, int fold_number,
      const ArrayList<index_t>& permutation,
      Dataset *train, Dataset *test) const;
};

namespace data {
  success_t Load(const char *fname, Matrix *matrix);
  template<typename Precision>
  success_t LargeLoad(const char *fname, GenMatrix<Precision> *matrix) {
    TextLineReader *reader = new TextLineReader();
    if (reader->Open(fname)==SUCCESS_FAIL) {
      reader->Error("Couldn't open %s", fname);
      return SUCCESS_FAIL;
    } 
    index_t dimension=0;
    String line=reader->Peek();
    ArrayList<String> result;
    result.Init();
    line.Split(",", &result);
    dimension=result.size();
    while (reader->Gobble()) {
    }
    matrix->StaticInit(dimension, reader->line_num());
    matrix->SetAll(0.0);
    delete reader;
    reader = new TextLineReader();
    reader->Open(fname);
    while (true) {
      String line=reader->Peek();
      ArrayList<String> result;
      result.Init();
      line.Split(",", &result);
      for(index_t i=0; i<result.size(); i++) {
        Precision num;
        sscanf(result[i].c_str(), "%lf", &num);
        matrix->set(i, reader->line_num()-1, (Precision)num);
      }
      if (reader->Gobble()==false) {
        break;
      }
    }
    return SUCCESS_PASS;  
  }


  success_t Save(const char *fname, const Matrix& matrix);
};

#endif