import: data frames are here!

Implemented a whole new class to represent the data that comes in from CSV and XLSX. See docs for more info.

Closes #153
This commit is contained in:
Andy Meneely
2016-12-02 00:17:57 -05:00
parent 935040b342
commit f4d94240e0
9 changed files with 484 additions and 30 deletions
+6 -5
View File
@@ -3,6 +3,7 @@ require 'csv'
require_relative '../args/input_file'
require_relative '../args/import'
require_relative '../args/csv_opts'
require_relative '../import/data_frame'
module Squib
@@ -12,7 +13,7 @@ module Squib
import = Args::Import.new.load!(opts)
s = Roo::Excelx.new(input.file[0])
s.default_sheet = s.sheets[input.sheet[0]]
data = {}
data = Squib::DataFrame.new
s.first_column.upto(s.last_column) do |col|
header = s.cell(s.first_row, col).to_s
header.strip! if import.strip?
@@ -39,14 +40,14 @@ module Squib
csv_opts = Args::CSV_Opts.new(opts)
table = CSV.parse(data, csv_opts.to_hash)
check_duplicate_csv_headers(table)
hash = Hash.new
hash = Squib::DataFrame.new
table.headers.each do |header|
new_header = header.to_s
new_header = new_header.strip if import.strip?
hash[new_header] ||= table[header]
end
if import.strip?
new_hash = Hash.new
new_hash = Squib::DataFrame.new
hash.each do |header, col|
new_hash[header] = col.map do |str|
str = str.strip if str.respond_to?(:strip)
@@ -78,9 +79,9 @@ module Squib
# @api private
def explode_quantities(data, qty)
return data unless data.key? qty.to_s.strip
return data unless data.col? qty.to_s.strip
qtys = data[qty]
new_data = {}
new_data = Squib::DataFrame.new
data.each do |col, arr|
new_data[col] = []
qtys.each_with_index do |qty, index|
+108
View File
@@ -0,0 +1,108 @@
# encoding: UTF-8
require 'json'
require 'forwardable'
module Squib
class DataFrame
include Enumerable
def initialize(hash = {}, def_columns = true)
@hash = hash
columns.each { |col| def_column(col) } if def_columns
end
def each(&block)
@hash.each(&block)
end
def [](i)
@hash[i]
end
def []=(col, v)
@hash[col] = v
def_column(col)
return v
end
def columns
@hash.keys
end
def ncolumns
@hash.keys.size
end
def col?(col)
@hash.key? col
end
def row(i)
@hash.inject(Hash.new) { |ret, (name, arr)| ret[name] = arr[i]; ret }
end
def nrows
@hash.inject(0) { |max, (_n, col)| col.size > max ? col.size : max }
end
def to_json
@hash.to_json
end
def to_pretty_json
JSON.pretty_generate(@hash)
end
def to_h
@hash
end
def to_pretty_text
max_col = columns.inject(0) { |max, c | c.length > max ? c.length : max }
top = " ╭#{'-' * 36}╮\n"
bottom = " ╰#{'-' * 36}╯\n"
str = ''
0.upto(nrows - 1) do | i |
str += (' ' * max_col) + top
row(i).each do |col, data|
str += "#{col.rjust(max_col)} #{wrap_n_pad(data, max_col)}"
end
str += (' ' * max_col) + bottom
end
return str
end
private
def snake_case(str)
str.to_s.
strip.
gsub(/\s+/,'_').
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
gsub(/([a-z]+)([A-Z])/,'\1_\2').
downcase.
to_sym
end
def wrap_n_pad(str, max_col)
str.to_s.
concat(' '). # handle nil & empty strings
scan(/.{1,34}/).
map { |s| (' ' * max_col) + " | " + s.ljust(34) }.
join(" |\n").
lstrip. # initially no whitespace next to key
concat(" |\n")
end
def def_column(col)
raise "Column #{col} - does not exist" unless @hash.key? col
method_name = snake_case(col)
return if self.class.method_defined?(method_name) #warn people? or skip?
define_singleton_method method_name do
@hash[col]
end
end
end
end