From 5d3ead1284e749256500ca8eb57ebdfac8e8ac7e Mon Sep 17 00:00:00 2001 From: Andy Meneely Date: Fri, 2 Oct 2020 14:18:08 -0400 Subject: [PATCH] ported csv and yaml over --- lib/squib/api/data.rb | 100 ------------------------------ lib/squib/args/import.rb | 14 ++++- lib/squib/deck.rb | 3 +- lib/squib/dsl/csv.rb | 41 ++++++++++++ lib/squib/dsl/xlsx.rb | 1 - lib/squib/dsl/yaml.rb | 40 ++++++++++++ lib/squib/import/csv_importer.rb | 44 +++++++++++++ lib/squib/import/yaml_importer.rb | 29 +++++++++ 8 files changed, 167 insertions(+), 105 deletions(-) delete mode 100644 lib/squib/api/data.rb create mode 100644 lib/squib/dsl/csv.rb create mode 100644 lib/squib/dsl/yaml.rb create mode 100644 lib/squib/import/csv_importer.rb create mode 100644 lib/squib/import/yaml_importer.rb diff --git a/lib/squib/api/data.rb b/lib/squib/api/data.rb deleted file mode 100644 index 73fa96a..0000000 --- a/lib/squib/api/data.rb +++ /dev/null @@ -1,100 +0,0 @@ -require 'roo' -require 'csv' -require 'yaml' -require_relative '../args/input_file' -require_relative '../args/import' -require_relative '../args/csv_opts' -require_relative '../import/data_frame' - -module Squib - - # DSL method. See http://squib.readthedocs.io - def csv(opts = {}) - # TODO refactor all this out to separate methods, and its own class - import = Args::Import.new.load!(opts) - file = Args::InputFile.new(file: 'deck.csv').load!(opts).file[0] - data = opts.key?(:data) ? opts[:data] : File.read(file) - csv_opts = Args::CSV_Opts.new(opts) - table = CSV.parse(data, **csv_opts.to_hash) - check_duplicate_csv_headers(table) - hash = Squib::DataFrame.new - table.headers.each do |header| - new_header = header.to_s - new_header = new_header.strip if import.strip? - hash[new_header] ||= table[header] - end - if import.strip? - new_hash = Squib::DataFrame.new - hash.each do |header, col| - new_hash[header] = col.map do |str| - str = str.strip if str.respond_to?(:strip) - str - end - end - hash = new_hash - end - if block_given? - hash.each do |header, col| - col.map! do |val| - yield(header, val) - end - end - end - return explode_quantities(hash, import.explode) - end - module_function :csv - - # DSL method. See http://squib.readthedocs.io - def yaml(opts = {}) - input = Args::InputFile.new(file: 'deck.yml').load!(opts) - import = Args::Import.new.load!(opts) - yml = YAML.load_file(input.file[0]) - data = Squib::DataFrame.new - # Get a universal list of keys to ensure everything is covered. - keys = yml.map { |c| c.keys}.flatten.uniq - keys.each { |k| data[k] = [] } #init arrays - yml.each do |card| - # nil value if key isn't set. - keys.each { |k| data[k] << card[k] } - end - if block_given? - data.each do |header, col| - col.map! do |val| - yield(header, val) - end - end - end - explode_quantities(data, import.explode) - end - module_function :yaml - - # Check if the given CSV table has duplicate columns, and throw a warning - # @api private - def check_duplicate_csv_headers(table) - if table.headers.size != table.headers.uniq.size - dups = table.headers.select{|e| table.headers.count(e) > 1 } - Squib.logger.warn "CSV duplicated the following column keys: #{dups.join(',')}" - end - end - module_function :check_duplicate_csv_headers - - - - class Deck - - # DSL method. See http://squib.readthedocs.io - def xlsx(opts = {}) - Squib.xlsx(opts) - end - - # DSL method. See http://squib.readthedocs.io - def csv(opts = {}) - Squib.csv(opts) - end - - # DSL method. See http://squib.readthedocs.io - def yaml(opts = {}) - Squib.yaml(opts) - end - end -end diff --git a/lib/squib/args/import.rb b/lib/squib/args/import.rb index 10c61de..2c0bc25 100644 --- a/lib/squib/args/import.rb +++ b/lib/squib/args/import.rb @@ -10,10 +10,12 @@ module Squib::Args class Import def self.parameters - { strip: true, + { + data: nil, explode: 'qty', file: nil, - sheet: 0 + sheet: 0, + strip: true, } end @@ -31,6 +33,7 @@ module Squib::Args @strip = validate_strip opts[:strip] @explode = validate_explode opts[:explode] @file = validate_file opts[:file] + @data = validate_data opts[:data] @sheet = opts[:sheet] return self end @@ -45,7 +48,7 @@ module Squib::Args end def validate_file(arg) - raise 'file argument not provided.' if arg.nil? + return nil if arg.nil? raise "File #{File.expand_path(arg)} does not exist!" unless File.exists?(arg) File.expand_path(arg) end @@ -54,6 +57,11 @@ module Squib::Args strip end + def validate_data(arg) + return nil if arg.nil? + arg.to_s + end + end end diff --git a/lib/squib/deck.rb b/lib/squib/deck.rb index 5a2e03d..bad3ae3 100644 --- a/lib/squib/deck.rb +++ b/lib/squib/deck.rb @@ -102,7 +102,6 @@ module Squib ################## ### PUBLIC API ### ################## - require_relative 'api/data' require_relative 'api/settings' ################### @@ -110,6 +109,7 @@ module Squib ################### require_relative 'dsl/background' require_relative 'dsl/circle' + require_relative 'dsl/csv' require_relative 'dsl/curve' require_relative 'dsl/cut_zone' require_relative 'dsl/ellipse' @@ -132,5 +132,6 @@ module Squib require_relative 'dsl/triangle' require_relative 'dsl/units' require_relative 'dsl/xlsx' + require_relative 'dsl/yaml' end end diff --git a/lib/squib/dsl/csv.rb b/lib/squib/dsl/csv.rb new file mode 100644 index 0000000..5194429 --- /dev/null +++ b/lib/squib/dsl/csv.rb @@ -0,0 +1,41 @@ +require_relative '../args/import' +require_relative '../import/csv_importer' +require_relative '../errors_warnings/warn_unexpected_params' + +module Squib + # DSL method. See http://squib.readthedocs.io + def csv(opts = {}, &block) + DSL::Csv.new(__callee__).run(opts, &block) + end + module_function :csv + + class Deck + # DSL method. See http://squib.readthedocs.io + def csv(opts = {}, &block) + DSL::Csv.new(__callee__).run(opts, &block) + end + end + + module DSL + class Csv + include WarnUnexpectedParams + attr_reader :dsl_method, :block + + def initialize(dsl_method) + @dsl_method = dsl_method + end + + def self.accepted_params + %i( file data strip explode col_sep quote_char) + end + + def run(opts,&block) + warn_if_unexpected opts + import_args = Args.extract_import opts + importer = Squib::Import::CsvImporter.new + csv_opts = Args::CSV_Opts.new(opts) + importer.import_to_dataframe(import_args, csv_opts, &block) + end + end + end +end diff --git a/lib/squib/dsl/xlsx.rb b/lib/squib/dsl/xlsx.rb index dd54f59..f0f1eb5 100644 --- a/lib/squib/dsl/xlsx.rb +++ b/lib/squib/dsl/xlsx.rb @@ -1,4 +1,3 @@ -require_relative '../args/input_file' require_relative '../args/import' require_relative '../import/xlsx_importer' require_relative '../errors_warnings/warn_unexpected_params' diff --git a/lib/squib/dsl/yaml.rb b/lib/squib/dsl/yaml.rb new file mode 100644 index 0000000..df4ea61 --- /dev/null +++ b/lib/squib/dsl/yaml.rb @@ -0,0 +1,40 @@ +require_relative '../args/import' +require_relative '../import/Yaml_importer' +require_relative '../errors_warnings/warn_unexpected_params' + +module Squib + # DSL method. See http://squib.readthedocs.io + def yaml(opts = {}, &block) + DSL::Yaml.new(__callee__).run(opts, &block) + end + module_function :yaml + + class Deck + # DSL method. See http://squib.readthedocs.io + def yaml(opts = {}, &block) + DSL::Yaml.new(__callee__).run(opts, &block) + end + end + + module DSL + class Yaml + include WarnUnexpectedParams + attr_reader :dsl_method, :block + + def initialize(dsl_method) + @dsl_method = dsl_method + end + + def self.accepted_params + %i( file data explode ) + end + + def run(opts,&block) + warn_if_unexpected opts + import_args = Args.extract_import opts + importer = Squib::Import::YamlImporter.new + importer.import_to_dataframe(import_args, &block) + end + end + end +end diff --git a/lib/squib/import/csv_importer.rb b/lib/squib/import/csv_importer.rb new file mode 100644 index 0000000..3e59a69 --- /dev/null +++ b/lib/squib/import/csv_importer.rb @@ -0,0 +1,44 @@ +require_relative 'quantity_exploder' + +module Squib::Import + class CsvImporter + include Squib::Import::QuantityExploder + def import_to_dataframe(import, csv_opts, &block) + data = import.data.nil? ? File.read(import.file) : import.data + table = CSV.parse(data, **csv_opts.to_hash) + check_duplicate_csv_headers(table) + hash = Squib::DataFrame.new + table.headers.each do |header| + new_header = header.to_s + new_header = new_header.strip if import.strip? + hash[new_header] ||= table[header] + end + if import.strip? + new_hash = Squib::DataFrame.new + hash.each do |header, col| + new_hash[header] = col.map do |str| + str = str.strip if str.respond_to?(:strip) + str + end + end + hash = new_hash + end + unless block.nil? + hash.each do |header, col| + col.map! do |val| + yield(header, val) + end + end + end + return explode_quantities(hash, import.explode) + end + + def check_duplicate_csv_headers(table) + if table.headers.size != table.headers.uniq.size + dups = table.headers.select{|e| table.headers.count(e) > 1 } + Squib.logger.warn "CSV duplicated the following column keys: #{dups.join(',')}" + end + end + end +end + diff --git a/lib/squib/import/yaml_importer.rb b/lib/squib/import/yaml_importer.rb new file mode 100644 index 0000000..7620392 --- /dev/null +++ b/lib/squib/import/yaml_importer.rb @@ -0,0 +1,29 @@ +require_relative 'data_frame' +require_relative 'quantity_exploder' + +module Squib::Import + class YamlImporter + include Squib::Import::QuantityExploder + def import_to_dataframe(import, &block) + data = import.data.nil? ? File.read(import.file) : import.data + yml = YAML.load(data) + data = Squib::DataFrame.new + # Get a universal list of keys to ensure everything is covered. + keys = yml.map { |c| c.keys}.flatten.uniq + keys.each { |k| data[k] = [] } #init arrays + yml.each do |card| + # nil value if key isn't set. + keys.each { |k| data[k] << card[k] } + end + unless block.nil? + data.each do |header, col| + col.map! do |val| + block.yield(header, val) + end + end + end + explode_quantities(data, import.explode) + end + end +end +