SparseData.h

Go to the documentation of this file.

//===========================================================================
/*!
 * 
 *
 * \brief   Support for importing and exporting data from and to sparse data (libSVM) formatted data files
 * 
 * 
 * \par
 * The most important application of the methods provided in this
 * file is the import of data from LIBSVM files to Shark Data containers.
 * 
 * 
 * 
 *
 * \author      M. Tuma, T. Glasmachers, C. Igel
 * \date        2010-2016
 *
 *
 * \par Copyright 1995-2017 Shark Development Team
 * 
 * <BR><HR>
 * This file is part of Shark.
 * <https://shark-ml.github.io/Shark/>
 * 
 * Shark is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published 
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * Shark is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with Shark.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
//===========================================================================
 
#ifndef SHARK_DATA_SPARSEDATA_H
#define SHARK_DATA_SPARSEDATA_H
 
#include <shark/Core/DLLSupport.h>
#include <shark/Core/utility/KeyValuePair.h>
#include <shark/Data/Dataset.h>
#include <fstream>
 
namespace shark {
 
/**
 * \ingroup shark_globals
 *
 * @{
 */
 
 
 
/// \brief Import classification data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  stream        stream to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<RealVector, unsigned int>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
 
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<FloatVector, unsigned int>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
 
/// \brief Import regression data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  stream        stream to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<RealVector, RealVector>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
 
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<FloatVector, FloatVector>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
 
/// \brief Import classification data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  stream        stream to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedRealVector, unsigned int>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedFloatVector, unsigned int>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
 
/// \brief Import regression data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  stream        stream to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedRealVector, RealVector>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedFloatVector, FloatVector>& dataset,
    std::istream& stream,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
 
 
/// \brief Import classification data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  fn            the file to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<RealVector, unsigned int>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<FloatVector, unsigned int>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
 
/// \brief Import regression data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  fn            the file to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<RealVector, RealVector>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<FloatVector, FloatVector>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
 
/// \brief Import classification data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  fn            the file to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedRealVector, unsigned int>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedFloatVector, unsigned int>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, unsigned int>::DefaultBatchSize
);
 
/// \brief Import regression data from a sparse data (libSVM) file.
///
/// \param  dataset       container storing the loaded data
/// \param  fn            the file to be read from
/// \param  highestIndex  highest feature index, or 0 for auto-detection
/// \param  batchSize     size of batch
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedRealVector, RealVector>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
SHARK_EXPORT_SYMBOL void importSparseData(
    LabeledData<CompressedFloatVector, FloatVector>& dataset,
    std::string fn,
    unsigned int highestIndex = 0,
    std::size_t batchSize = LabeledData<RealVector, RealVector>::DefaultBatchSize
);
 
 
/// \brief Export classification data to sparse data (libSVM) format.
///
/// \param  dataset     Container storing the  data
/// \param  stream      Output stream
/// \param  oneMinusOne Flag for applying the transformation y<-2y-1 to binary labels
/// \param  sortLabels  Flag for sorting data points according to labels
template<typename InputType>
void exportSparseData(LabeledData<InputType, unsigned int> const& dataset, std::ostream& stream, bool oneMinusOne = true, bool sortLabels = false)
{
    if (numberOfClasses(dataset) != 2) oneMinusOne = false;
 
    std::vector< KeyValuePair<unsigned int, std::pair<std::size_t, std::size_t> > > order;
    for (std::size_t b=0; b<dataset.numberOfBatches(); b++)
    {
        auto batch = dataset.batch(b);
        for (std::size_t i=0; i<batchSize(batch); i++)
        {
            order.emplace_back(getBatchElement(batch, i).label, std::make_pair(b, i));
        }
    }
    if (sortLabels)
    {
        std::sort(order.begin(), order.end());
    }
 
    for (auto const& p : order)
    {
        auto element = getBatchElement(dataset.batch(p.value.first), p.value.second);
        // apply transformation to label and write it to file
        if (oneMinusOne) stream << 2*int(element.label)-1 << " ";
        //libsvm file format documentation is scarce, but by convention the first class seems to be 1..
        else stream << element.label+1 << " ";
        // write input data to file
        for (auto it = element.input.begin(); it != element.input.end(); ++it)
        {
            stream << " " << it.index()+1 << ":" << *it;
        }
        stream << std::endl;
    }
}
 
/// \brief Export classification data to sparse data (libSVM) format.
///
/// \param  dataset     Container storing the data
/// \param  fn          Output file name
/// \param  oneMinusOne Flag for applying the transformation y<-2y-1 to binary labels
/// \param  sortLabels  Flag for sorting data points according to labels
/// \param  append      Flag for appending to the output file instead of overwriting it
template<typename InputType>
void exportSparseData(LabeledData<InputType, unsigned int> const& dataset, const std::string &fn, bool oneMinusOne = true, bool sortLabels = false, bool append = false)
{
    std::ofstream ofs;
 
    // shall we append only or overwrite?
    if (append == true) {
        ofs.open (fn.c_str(), std::fstream::out | std::fstream::app );
    } else {
        ofs.open (fn.c_str());
    }
    SHARK_RUNTIME_CHECK(ofs, "File can not be opened for writing");
 
    exportSparseData(dataset, ofs, oneMinusOne, sortLabels);
}
 
/// \brief Export regression data to sparse data (libSVM) format.
///
/// \param  dataset     Container storing the data
/// \param  stream      Output stream
template<typename InputType>
void exportSparseData(LabeledData<InputType, RealVector> const& dataset, std::ostream& stream)
{
    for (std::size_t b=0; b<dataset.numberOfBatches(); b++)
    {
        auto batch = dataset.batch(b);
        for (std::size_t i=0; i<batchSize(batch); i++)
        {
            auto element = getBatchElement(batch, i);
            SHARK_ASSERT(element.label.size() == 1);
            stream << element.label(0);
            for (auto it = element.input.begin(); it != element.input.end(); ++it)
            {
                stream << " " << it.index()+1 << ":" << *it;
            }
            stream << std::endl;
        }
    }
}
 
/// \brief Export regression data to sparse data (libSVM) format.
///
/// \param  dataset     Container storing the  data
/// \param  fn          Output file
/// \param  append      Flag for appending to the output file instead of overwriting it
template<typename InputType>
void exportSparseData(LabeledData<InputType, RealVector> const& dataset, const std::string &fn, bool append = false)
{
    std::ofstream ofs;
 
    // shall we append only or overwrite?
    if (append == true) {
        ofs.open (fn.c_str(), std::fstream::out | std::fstream::app );
    } else {
        ofs.open (fn.c_str());
    }
 
    SHARK_RUNTIME_CHECK(ofs, "File can not be opened for writing");
 
    exportSparseData(dataset, ofs);
}
 
/** @}*/
 
}
#endif