Anshorei
/
squib
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ported csv and yaml over

dev
Andy Meneely 2020-10-02 14:18:08 -04:00
parent 9b0602a50c
commit 5d3ead1284
8 changed files with 167 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
lib/squib/api/data.rb
View File

@ -1,100 +0,0 @@
require 'roo'
require 'csv'
require 'yaml'
require_relative '../args/input_file'
require_relative '../args/import'
require_relative '../args/csv_opts'
require_relative '../import/data_frame'
module Squib
# DSL method. See http://squib.readthedocs.io
def csv(opts = {})
# TODO refactor all this out to separate methods, and its own class
import = Args::Import.new.load!(opts)
file = Args::InputFile.new(file: 'deck.csv').load!(opts).file[0]
data = opts.key?(:data) ? opts[:data] : File.read(file)
csv_opts = Args::CSV_Opts.new(opts)
table = CSV.parse(data, **csv_opts.to_hash)
check_duplicate_csv_headers(table)
hash = Squib::DataFrame.new
table.headers.each do |header|
new_header = header.to_s
new_header = new_header.strip if import.strip?
hash[new_header] ||= table[header]
end
if import.strip?
new_hash = Squib::DataFrame.new
hash.each do |header, col|
new_hash[header] = col.map do |str|
str = str.strip if str.respond_to?(:strip)
str
end
end
hash = new_hash
end
if block_given?
hash.each do |header, col|
col.map! do |val|
yield(header, val)
end
end
end
return explode_quantities(hash, import.explode)
end
module_function :csv
# DSL method. See http://squib.readthedocs.io
def yaml(opts = {})
input = Args::InputFile.new(file: 'deck.yml').load!(opts)
import = Args::Import.new.load!(opts)
yml = YAML.load_file(input.file[0])
data = Squib::DataFrame.new
# Get a universal list of keys to ensure everything is covered.
keys = yml.map { |c| c.keys}.flatten.uniq
keys.each { |k| data[k] = [] } #init arrays
yml.each do |card|
# nil value if key isn't set.
keys.each { |k| data[k] << card[k] }
end
if block_given?
data.each do |header, col|
col.map! do |val|
yield(header, val)
end
end
end
explode_quantities(data, import.explode)
end
module_function :yaml
# Check if the given CSV table has duplicate columns, and throw a warning
# @api private
def check_duplicate_csv_headers(table)
if table.headers.size != table.headers.uniq.size
dups = table.headers.select{|e| table.headers.count(e) > 1 }
Squib.logger.warn "CSV duplicated the following column keys: #{dups.join(',')}"
end
end
module_function :check_duplicate_csv_headers
class Deck
# DSL method. See http://squib.readthedocs.io
def xlsx(opts = {})
Squib.xlsx(opts)
end
# DSL method. See http://squib.readthedocs.io
def csv(opts = {})
Squib.csv(opts)
end
# DSL method. See http://squib.readthedocs.io
def yaml(opts = {})
Squib.yaml(opts)
end
end
end

14
lib/squib/args/import.rb
View File

@ -10,10 +10,12 @@ module Squib::Args
class Import
def self.parameters
{ strip: true,
{
data: nil,
explode: 'qty',
file: nil,
sheet: 0
sheet: 0,
strip: true,
}
end
@ -31,6 +33,7 @@ module Squib::Args
@strip = validate_strip opts[:strip]
@explode = validate_explode opts[:explode]
@file = validate_file opts[:file]
@data = validate_data opts[:data]
@sheet = opts[:sheet]
return self
end
@ -45,7 +48,7 @@ module Squib::Args
end
def validate_file(arg)
raise 'file argument not provided.' if arg.nil?
return nil if arg.nil?
raise "File #{File.expand_path(arg)} does not exist!" unless File.exists?(arg)
File.expand_path(arg)
end
@ -54,6 +57,11 @@ module Squib::Args
strip
end
def validate_data(arg)
return nil if arg.nil?
arg.to_s
end
end
end

3
lib/squib/deck.rb
View File

@ -102,7 +102,6 @@ module Squib
##################
### PUBLIC API ###
##################
require_relative 'api/data'
require_relative 'api/settings'
###################
@ -110,6 +109,7 @@ module Squib
###################
require_relative 'dsl/background'
require_relative 'dsl/circle'
require_relative 'dsl/csv'
require_relative 'dsl/curve'
require_relative 'dsl/cut_zone'
require_relative 'dsl/ellipse'
@ -132,5 +132,6 @@ module Squib
require_relative 'dsl/triangle'
require_relative 'dsl/units'
require_relative 'dsl/xlsx'
require_relative 'dsl/yaml'
end
end

41
lib/squib/dsl/csv.rb Normal file
View File

@ -0,0 +1,41 @@
require_relative '../args/import'
require_relative '../import/csv_importer'
require_relative '../errors_warnings/warn_unexpected_params'
module Squib
# DSL method. See http://squib.readthedocs.io
def csv(opts = {}, &block)
DSL::Csv.new(__callee__).run(opts, &block)
end
module_function :csv
class Deck
# DSL method. See http://squib.readthedocs.io
def csv(opts = {}, &block)
DSL::Csv.new(__callee__).run(opts, &block)
end
end
module DSL
class Csv
include WarnUnexpectedParams
attr_reader :dsl_method, :block
def initialize(dsl_method)
@dsl_method = dsl_method
end
def self.accepted_params
%i( file data strip explode col_sep quote_char)
end
def run(opts,&block)
warn_if_unexpected opts
import_args = Args.extract_import opts
importer = Squib::Import::CsvImporter.new
csv_opts = Args::CSV_Opts.new(opts)
importer.import_to_dataframe(import_args, csv_opts, &block)
end
end
end
end

1
lib/squib/dsl/xlsx.rb
View File

@ -1,4 +1,3 @@
require_relative '../args/input_file'
require_relative '../args/import'
require_relative '../import/xlsx_importer'
require_relative '../errors_warnings/warn_unexpected_params'

40
lib/squib/dsl/yaml.rb Normal file
View File

@ -0,0 +1,40 @@
require_relative '../args/import'
require_relative '../import/Yaml_importer'
require_relative '../errors_warnings/warn_unexpected_params'
module Squib
# DSL method. See http://squib.readthedocs.io
def yaml(opts = {}, &block)
DSL::Yaml.new(__callee__).run(opts, &block)
end
module_function :yaml
class Deck
# DSL method. See http://squib.readthedocs.io
def yaml(opts = {}, &block)
DSL::Yaml.new(__callee__).run(opts, &block)
end
end
module DSL
class Yaml
include WarnUnexpectedParams
attr_reader :dsl_method, :block
def initialize(dsl_method)
@dsl_method = dsl_method
end
def self.accepted_params
%i( file data explode )
end
def run(opts,&block)
warn_if_unexpected opts
import_args = Args.extract_import opts
importer = Squib::Import::YamlImporter.new
importer.import_to_dataframe(import_args, &block)
end
end
end
end

44
lib/squib/import/csv_importer.rb Normal file
View File

@ -0,0 +1,44 @@
require_relative 'quantity_exploder'
module Squib::Import
class CsvImporter
include Squib::Import::QuantityExploder
def import_to_dataframe(import, csv_opts, &block)
data = import.data.nil? ? File.read(import.file) : import.data
table = CSV.parse(data, **csv_opts.to_hash)
check_duplicate_csv_headers(table)
hash = Squib::DataFrame.new
table.headers.each do |header|
new_header = header.to_s
new_header = new_header.strip if import.strip?
hash[new_header] ||= table[header]
end
if import.strip?
new_hash = Squib::DataFrame.new
hash.each do |header, col|
new_hash[header] = col.map do |str|
str = str.strip if str.respond_to?(:strip)
str
end
end
hash = new_hash
end
unless block.nil?
hash.each do |header, col|
col.map! do |val|
yield(header, val)
end
end
end
return explode_quantities(hash, import.explode)
end
def check_duplicate_csv_headers(table)
if table.headers.size != table.headers.uniq.size
dups = table.headers.select{|e| table.headers.count(e) > 1 }
Squib.logger.warn "CSV duplicated the following column keys: #{dups.join(',')}"
end
end
end
end

29
lib/squib/import/yaml_importer.rb Normal file
View File

@ -0,0 +1,29 @@
require_relative 'data_frame'
require_relative 'quantity_exploder'
module Squib::Import
class YamlImporter
include Squib::Import::QuantityExploder
def import_to_dataframe(import, &block)
data = import.data.nil? ? File.read(import.file) : import.data
yml = YAML.load(data)
data = Squib::DataFrame.new
# Get a universal list of keys to ensure everything is covered.
keys = yml.map { |c| c.keys}.flatten.uniq
keys.each { |k| data[k] = [] } #init arrays
yml.each do |card|
# nil value if key isn't set.
keys.each { |k| data[k] << card[k] }
end
unless block.nil?
data.each do |header, col|
col.map! do |val|
block.yield(header, val)
end
end
end
explode_quantities(data, import.explode)
end
end
end