dataframe.hpp

/****************************************************************************
 *  hipipe library
 *  Copyright (c) 2017, Cognexa Solutions s.r.o.
 *  Copyright (c) 2018, Iterait a.s.
 *  Author(s) Filip Matzner
 *
 *  This file is distributed under the MIT License.
 *  See the accompanying file LICENSE.txt for the complete license agreement.
 ****************************************************************************/
 
#pragma once
 
#include <hipipe/core/index_mapper.hpp>
#include <hipipe/core/utility/string.hpp>
#include <hipipe/core/utility/tuple.hpp>
 
#include <range/v3/experimental/view/shared.hpp>
#include <range/v3/view/all.hpp>
#include <range/v3/view/iota.hpp>
#include <range/v3/view/move.hpp>
#include <range/v3/view/transform.hpp>
#include <range/v3/view/zip.hpp>
#include <range/v3/view/zip_with.hpp>
 
#include <functional>
#include <iomanip>
#include <iostream>
#include <vector>
 
namespace hipipe {
 
namespace rgv = ranges::views;
 
class dataframe {
public:
 
    using data_table_t = std::vector<std::vector<std::string>>;
    using header_t = index_mapper<std::string>;
 
private:
 
    // data storage //
 
    data_table_t data_;
    header_t header_;
 
    // helper functions //
 
    static void throw_check_new_header(
      std::size_t n_cols,
      const std::vector<std::string>& header)
    {
        if (header.size() && header.size() != n_cols) {
            throw std::invalid_argument{"The dataframe with " + std::to_string(n_cols) +
              " columns cannot have a header of size " + std::to_string(header.size()) + "."};
        }
        for (const std::string& h : header) {
            if (!h.size()) {
                throw std::invalid_argument{"When providing a header to a dataframe,"
                  " all the column names have to be non-empty."};
            }
        }
    }
 
    void throw_check_insert_col_name(const std::string& name) const
    {
        if (header_.size() && !name.size()) {
            throw std::invalid_argument{"The dataframe has a header, please provide"
              " a column name when inserting a new column."};
        }
        if (n_cols() != 0 && !header_.size() && name.size()) {
            throw std::invalid_argument{"The dataframe has no header, but a column"
              " name \"" + name + "\" was provided when inserting a new column."};
        }
    }
 
    void throw_check_insert_col_size(std::size_t col_size) const
    {
        if (n_rows() != 0 && col_size != n_rows()) {
            throw std::invalid_argument{"Cannot insert a column of size "
              + std::to_string(col_size) + " to a dataframe with "
              + std::to_string(n_rows()) + " rows."};
        }
    }
 
    void throw_check_insert_row_size(std::size_t row_size) const
    {
        if (n_cols() != 0 && row_size != n_cols()) {
            throw std::invalid_argument{"Cannot insert a row of size "
              + std::to_string(row_size) + " to a dataframe with "
              + std::to_string(n_cols()) + " columns."};
        }
    }
 
    void throw_check_row_idx(std::size_t row_idx) const
    {
        if (row_idx < 0 || row_idx >= n_rows()) {
            throw std::out_of_range{"Row index " + std::to_string(row_idx) +
              " is not in a dataframe with " + std::to_string(n_rows()) + " rows."};
        }
    }
 
    void throw_check_col_idx(std::size_t col_idx) const
    {
        if (col_idx < 0 || col_idx >= n_cols()) {
            throw std::out_of_range{"Column index " + std::to_string(col_idx) +
              " is not in a dataframe with " + std::to_string(n_cols()) + " columns."};
        }
    }
 
    void throw_check_col_name(const std::string& col_name) const
    {
        if (header_.size() == 0) {
            throw std::out_of_range{"Dataframe has no header, cannot index by column name."};
        }
        if (!header_.contains(col_name)) {
            throw std::out_of_range{"Column " + col_name + " not found in the dataframe."};
        }
    }
 
    template <typename This>
    static auto raw_irows_impl(This this_ptr, std::vector<std::size_t> col_indexes)
    {
        return rgv::iota(0UL, this_ptr->n_rows())
          | rgv::transform([this_ptr, col_indexes=std::move(col_indexes)](std::size_t i) {
                return this_ptr->raw_icols(col_indexes)
                  // decltype(auto) to make sure a reference is returned
                  | rgv::transform([i](auto&& col) -> decltype(auto) {
                        return col[i];
                    });
            });
      }
 
      template<typename This>
      static auto raw_rows_impl(This this_ptr)
      {
        return rgv::iota(0UL, this_ptr->n_rows())
          | rgv::transform([this_ptr](std::size_t i) {
                return rgv::iota(0UL, this_ptr->n_cols())
                  // decltype(auto) to make sure a reference is returned
                  | rgv::transform([this_ptr, i](std::size_t j) -> decltype(auto) {
                        return this_ptr->raw_cols()[j][i];
                    });
            });
      }
 
      template<typename This>
      static auto raw_icols_impl(This this_ptr, std::vector<std::size_t> col_indexes)
      {
        return std::move(col_indexes)
          | ranges::experimental::views::shared
          | rgv::transform([this_ptr](std::size_t idx) {
                return this_ptr->raw_cols()[idx];
            });
      }
 
public:
 
    dataframe() = default;
 
    template<typename T>
    dataframe(std::vector<std::vector<T>> columns, std::vector<std::string> header = {})
    {
        throw_check_new_header(columns.size(), header);
        for (std::size_t i = 0; i < columns.size(); ++i) {
            std::string col_name = header.empty() ? "" : std::move(header[i]);
            insert_col(rgv::move(columns[i]), std::move(col_name));
        }
    }
 
    template<typename... Ts>
    dataframe(std::tuple<std::vector<Ts>...> columns, std::vector<std::string> header = {})
    {
        throw_check_new_header(sizeof...(Ts), header);
        utility::tuple_for_each_with_index(std::move(columns),
          [this, &header](auto& column, auto index) {
              std::string col_name = header.empty() ? "" : std::move(header[index]);
              this->insert_col(rgv::move(column), std::move(col_name));
        });
    }
 
    // insertion //
 
    template<typename Rng, typename ValueT = ranges::range_value_t<Rng>>
    std::size_t insert_col(Rng&& rng, std::string col_name = {},
                           std::function<std::string(const ValueT&)> cvt =
                             static_cast<std::string (*)(const ValueT&)>(utility::to_string))
    {
        throw_check_insert_col_name(col_name);
        throw_check_insert_col_size(ranges::size(rng));
        if (col_name.size()) header_.insert(col_name);
        data_.emplace_back(rgv::transform(rng, cvt));
        return n_cols() - 1;
    }
 
    template<typename... Ts>
    std::size_t insert_row(std::tuple<Ts...> row_tuple,
                           std::tuple<std::function<std::string(const Ts&)>...> cvts =
                             std::make_tuple(
                               static_cast<std::string (*)(const Ts&)>(utility::to_string)...))
    {
        throw_check_insert_row_size(sizeof...(Ts));
        utility::tuple_for_each_with_index(std::move(row_tuple),
          [this, &cvts](auto& field, auto index) {
              this->data_.at(index).push_back(std::get<index>(cvts)(std::move(field)));
        });
        return n_rows() - 1;
    }
 
    std::size_t insert_row(std::vector<std::string> row)
    {
        throw_check_insert_row_size(row.size());
        for (std::size_t i = 0; i < n_cols(); ++i) {
            data_[i].push_back(std::move(row[i]));
        }
        return n_rows() - 1;
    }
 
    // drop //
 
    void drop_icol(std::size_t col_index)
    {
        throw_check_col_idx(col_index);
        // remove the column from the header
        if (header_.size()) {
            std::vector<std::string> new_header = header_.values();
            new_header.erase(new_header.begin() + col_index);
            header_ = new_header;
        }
        // remove the column from the data
        data_.erase(data_.begin() + col_index);
    }
 
    void drop_col(const std::string& col_name)
    {
        throw_check_col_name(col_name);
        return drop_icol(header_.index_for(col_name));
    }
 
    void drop_row(const std::size_t row_idx)
    {
        throw_check_row_idx(row_idx);
        for (auto& column : data_) {
            column.erase(column.begin() + row_idx);
        }
    }
 
    // raw multi column access //
 
    auto raw_cols()
    {
        return rgv::transform(data_, rgv::all);
    }
 
    auto raw_cols() const
    {
        return rgv::transform(data_, rgv::all);
    }
 
    auto raw_icols(std::vector<std::size_t> col_indexes)
    {
        for (auto& col_idx : col_indexes) throw_check_col_idx(col_idx);
        return raw_icols_impl(this, std::move(col_indexes));
    }
 
    auto raw_icols(std::vector<std::size_t> col_indexes) const
    {
        for (auto& col_idx : col_indexes) throw_check_col_idx(col_idx);
        return raw_icols_impl(this, std::move(col_indexes));
    }
 
    auto raw_cols(const std::vector<std::string>& col_names)
    {
        for (auto& col_name : col_names) throw_check_col_name(col_name);
        return raw_icols(header_.index_for(col_names));
    }
 
    auto raw_cols(const std::vector<std::string>& col_names) const
    {
        for (auto& col_name : col_names) throw_check_col_name(col_name);
        return raw_icols(header_.index_for(col_names));
    }
 
    // typed multi column access //
 
    template<typename... Ts>
    auto icols(std::vector<std::size_t> col_indexes,
               std::tuple<std::function<Ts(const std::string&)>...> cvts =
                 std::make_tuple(utility::string_to<Ts>...)) const
    {
        assert(sizeof...(Ts) == ranges::size(col_indexes));
        return utility::tuple_transform_with_index(std::move(cvts),
          [raw_cols = raw_icols(std::move(col_indexes))](auto&& cvt, auto i) {
              return rgv::transform(raw_cols[i], std::move(cvt));
        });
    }
 
    template<typename... Ts>
    auto cols(const std::vector<std::string>& col_names,
              std::tuple<std::function<Ts(const std::string&)>...> cvts =
                std::make_tuple(utility::string_to<Ts>...)) const
    {
        for (auto& col_name : col_names) throw_check_col_name(col_name);
        return icols<Ts...>(header_.index_for(col_names), std::move(cvts));
    }
 
    // raw column access //
 
    auto raw_icol(std::size_t col_index)
    {
        throw_check_col_idx(col_index);
        return rgv::all(raw_cols()[col_index]);
    }
 
    auto raw_icol(std::size_t col_index) const
    {
        throw_check_col_idx(col_index);
        return rgv::all(raw_cols()[col_index]);
    }
 
    auto raw_col(const std::string& col_name)
    {
        throw_check_col_name(col_name);
        return raw_icol(header_.index_for(col_name));
    }
 
    auto raw_col(const std::string& col_name) const
    {
        throw_check_col_name(col_name);
        return raw_icol(header_.index_for(col_name));
    }
 
    // typed column access //
 
    template<typename T>
    auto icol(std::size_t col_index,
              std::function<T(const std::string&)> cvt = utility::string_to<T>) const
    {
        return rgv::transform(raw_icol(col_index), cvt);
    }
 
    template<typename T>
    auto col(const std::string& col_name,
             std::function<T(const std::string&)> cvt = utility::string_to<T>) const
    {
        throw_check_col_name(col_name);
        return icol<T>(header_.index_for(col_name), std::move(cvt));
    }
 
 
    auto raw_rows()
    {
        return raw_rows_impl(this);
    }
 
    auto raw_rows() const
    {
        return raw_rows_impl(this);
    }
 
    auto raw_irows(std::vector<std::size_t> col_indexes)
    {
        for (auto& col_idx : col_indexes) throw_check_col_idx(col_idx);
        return raw_irows_impl(this, std::move(col_indexes));
    }
 
    auto raw_irows(std::vector<std::size_t> col_indexes) const
    {
        for (auto& col_idx : col_indexes) throw_check_col_idx(col_idx);
        return raw_irows_impl(this, std::move(col_indexes));
    }
 
    auto raw_rows(const std::vector<std::string>& col_names)
    {
        for (auto& col_name : col_names) throw_check_col_name(col_name);
        return raw_irows(header_.index_for(col_names));
    }
 
    auto raw_rows(const std::vector<std::string>& col_names) const
    {
        for (auto& col_name : col_names) throw_check_col_name(col_name);
        return raw_irows(header_.index_for(col_names));
    }
 
    // typed row access //
 
    template<typename... Ts>
    auto irows(std::vector<std::size_t> col_indexes,
               std::tuple<std::function<Ts(const std::string&)>...> cvts =
                 std::make_tuple(utility::string_to<Ts>...)) const
    {
        // Make sure the zip produces std::tuple, not ranges::common_tuple or similar.
        auto zip_as_std_tuple = [](auto&&... rngs) {
            auto std_tupler = [](Ts... ts) -> std::tuple<Ts...> { return {std::move(ts)...}; };
            return rgv::zip_with(std_tupler, std::forward<decltype(rngs)>(rngs)...);
        };
        return std::apply(
          zip_as_std_tuple,
          icols<Ts...>(std::move(col_indexes), std::move(cvts)));
    }
 
    template<typename... Ts>
    auto rows(const std::vector<std::string>& col_names,
              std::tuple<std::function<Ts(const std::string&)>...> cvts =
                std::make_tuple(utility::string_to<Ts>...)) const
    {
        for (auto& col_name : col_names) throw_check_col_name(col_name);
        return irows<Ts...>(header_.index_for(col_names), std::move(cvts));
    }
 
    // typed indexed single column access //
 
    template <typename IndexT, typename ColT>
    auto index_icol(std::size_t key_col_index,
                    std::size_t val_col_index,
                    std::function<IndexT(const std::string&)> key_col_cvt =
                      utility::string_to<IndexT>,
                    std::function<ColT(const std::string&)> val_col_cvt =
                      utility::string_to<ColT>) const
    {
        auto key_col = icol<IndexT>(key_col_index, std::move(key_col_cvt));
        auto val_col = icol<ColT>(val_col_index, std::move(val_col_cvt));
        return rgv::zip(key_col, val_col);
    }
 
    template<typename IndexT, typename ColT>
    auto index_col(const std::string& key_col_name,
                   const std::string& val_col_name,
                   std::function<IndexT(const std::string&)> key_col_cvt =
                     utility::string_to<IndexT>,
                   std::function<ColT(const std::string&)> val_col_cvt =
                     utility::string_to<ColT>) const
    {
        throw_check_col_name(key_col_name);
        throw_check_col_name(val_col_name);
        return index_icol(header_.index_for(key_col_name),
                          header_.index_for(val_col_name),
                          std::move(key_col_cvt),
                          std::move(val_col_cvt));
    }
 
    // typed indexed multiple column access //
 
    template<typename IndexT, typename... Ts>
    auto index_icols(std::size_t key_col_index,
                     std::vector<std::size_t> val_col_indexes,
                     std::function<IndexT(const std::string&)> key_col_cvt =
                       utility::string_to<IndexT>,
                     std::tuple<std::function<Ts(const std::string&)>...> val_col_cvts =
                       std::make_tuple(utility::string_to<Ts>...)) const
    {
        auto key_col = icol<IndexT>(key_col_index, std::move(key_col_cvt));
        auto val_cols = irows<Ts...>(std::move(val_col_indexes), std::move(val_col_cvts));
        return rgv::zip(key_col, val_cols);
    }
 
    template<typename IndexT, typename... Ts>
    auto index_cols(const std::string& key_col_name,
                    const std::vector<std::string>& val_col_names,
                    std::function<IndexT(const std::string&)> key_col_cvt =
                      utility::string_to<IndexT>,
                    std::tuple<std::function<Ts(const std::string&)>...> val_col_cvts =
                      std::make_tuple(utility::string_to<Ts>...)) const
    {
        throw_check_col_name(key_col_name);
        for (auto& col_name : val_col_names) throw_check_col_name(col_name);
        assert(header_.size() && "Dataframe has no header, cannot index by column name.");
        return index_icols(header_.index_for(key_col_name),
                           header_.index_for(val_col_names),
                           std::move(key_col_cvt),
                           std::move(val_col_cvts));
    }
 
    // shape functions //
 
    std::size_t n_cols() const
    {
        return data_.size();
    }
 
    std::size_t n_rows() const
    {
        if (n_cols() == 0) return 0;
        return data_.front().size();
    }
 
    void header(std::vector<std::string> new_header)
    {
        throw_check_new_header(n_cols(), new_header);
        header_ = std::move(new_header);
    }
 
    std::vector<std::string> header() const
    {
        return header_.values();
    }
 
    data_table_t& data()
    {
        return data_;
    }
 
    const data_table_t& data() const
    {
        return data_;
    }
 
};  // class dataframe
 
 
std::ostream& operator<<(std::ostream& out, const dataframe& df);
 
 
}  // end namespace hipipe