ported xlsx over

docs/dsl/csv.rst

@@ -52,6 +52,9 @@ col_sep
 CSV custom options in Ruby standard lib.
   All of the options in Ruby's std lib version of CSV are supported **except** ``headers`` is always ``true`` and ``converters`` is always set to ``:numeric``. See the `Ruby Docs <http://ruby-doc.org/stdlib-2.2.0/libdoc/csv/rdoc/CSV.html#method-c-new>`_ for information on the options.
 
+.. warning::
+  Data import methods such as ``xlsx`` and ``csv`` will not consult your layout file or follow the :doc:`/arrays` feature.
+
 Individual Pre-processing
 -------------------------
 

docs/dsl/xlsx.rst

@@ -32,6 +32,9 @@ explode
 
   Quantity explosion will be applied to the column this name. For example, rows in the csv with a ``'qty'`` of 3 will be duplicated 3 times.
 
+.. warning::
+  Data import methods such as ``xlsx`` and ``csv`` will not consult your layout file or follow the :doc:`/arrays` feature.
+
 Individual Pre-processing
 -------------------------
 

docs/dsl/yaml.rst

@@ -35,6 +35,8 @@ explode
 
   Quantity explosion will be applied to the column this name. For example, rows in the csv with a ``'qty'`` of 3 will be duplicated 3 times.
 
+.. warning::
+  Data import methods such as ``xlsx`` and ``csv`` will not consult your layout file or follow the :doc:`/arrays` feature.
 
 Individual Pre-processing
 -------------------------

lib/squib/api/data.rb

@@ -8,30 +8,6 @@ require_relative '../import/data_frame'
 
 module Squib
 
-  # DSL method. See http://squib.readthedocs.io
-  def xlsx(opts = {})
-    input = Args::InputFile.new(file: 'deck.xlsx').load!(opts)
-    import = Args::Import.new.load!(opts)
-    s = Roo::Excelx.new(input.file[0])
-    s.default_sheet = s.sheets[input.sheet[0]]
-    data = Squib::DataFrame.new
-    s.first_column.upto(s.last_column) do |col|
-      header = s.cell(s.first_row, col).to_s
-      header.strip! if import.strip?
-      data[header] = []
-      (s.first_row + 1).upto(s.last_row) do |row|
-        cell = s.cell(row, col)
-        # Roo hack for avoiding unnecessary .0's on whole integers (https://github.com/roo-rb/roo/issues/139)
-        cell = s.excelx_value(row, col) if s.excelx_type(row, col) == [:numeric_or_formula, 'General']
-        cell.strip! if cell.respond_to?(:strip) && import.strip?
-        cell = yield(header, cell) if block_given?
-        data[header] << cell
-      end# row
-    end# col
-    explode_quantities(data, import.explode)
-  end# xlsx
-  module_function :xlsx
-
   # DSL method. See http://squib.readthedocs.io
   def csv(opts = {})
     # TODO refactor all this out to separate methods, and its own class
@@ -102,20 +78,7 @@ module Squib
   end
   module_function :check_duplicate_csv_headers
 
-  # @api private
-  def explode_quantities(data, qty)
-    return data unless data.col? qty.to_s.strip
-    qtys = data[qty]
-    new_data = Squib::DataFrame.new
-    data.each do |col, arr|
-      new_data[col] = []
-      qtys.each_with_index do |qty, index|
-        qty.to_i.times { new_data[col] << arr[index] }
-      end
-    end
-    return new_data
-  end
-  module_function :explode_quantities
+  
 
   class Deck
 

lib/squib/args/arg_loader.rb

@@ -12,7 +12,7 @@ module Squib::Args::ArgLoader
     load!(args, expand_by: deck.size, layout: deck.layout, dpi: deck.dpi)
   end
 
-  # Main class invoked by the client (i.e. api/ methods)
+  # Main class invoked by the client (i.e. dsl/ methods)
   def load!(args, expand_by: 1, layout: {}, dpi: 300)
     @dpi = dpi
     args[:layout] = prep_layout_args(args[:layout], expand_by: expand_by)

lib/squib/args/import.rb

@@ -1,16 +1,24 @@
-require_relative 'arg_loader'
-
 module Squib::Args
+  module_function def extract_import(opts)
+    # note how we don't use ArgLoader here because it's way more complex than
+    # what we need here. Don't need layouts or singleton expansion, so...
+    # ...let's just do it ourselves.
+    Import.parameters.each  { |p, value| opts[p] = value unless opts.key? p }
+    return Import.new.load! opts
+  end
 
   class Import
-    include ArgLoader
-
+    
     def self.parameters
       { strip: true,
-        explode: 'qty'
+        explode: 'qty',
+        file: nil,
+        sheet: 0
       }
     end
 
+    attr_accessor *(self.parameters.keys)
+
     def self.expanding_parameters
       [] # none of them
     end
@@ -19,13 +27,27 @@ module Squib::Args
       [] # none of them
     end
 
+    def load!(opts)
+      @strip   = validate_strip   opts[:strip]
+      @explode = validate_explode opts[:explode]
+      @file    = validate_file    opts[:file]
+      @sheet   = opts[:sheet]
+      return self
+    end
+
     def validate_strip(arg)
       raise 'Strip must be true or false' unless arg == true || arg == false
       arg
     end
 
     def validate_explode(arg)
-      arg
+      arg.to_s
+    end
+
+    def validate_file(arg)
+      raise 'file argument not provided.' if arg.nil?
+      raise "File #{File.expand_path(arg)} does not exist!" unless File.exists?(arg)
+      File.expand_path(arg)
     end
 
     def strip?

lib/squib/args/input_file.rb

@@ -13,9 +13,7 @@ module Squib::Args
     end
 
     def self.parameters
-      { file: nil,
-        sheet: 0,
-      }
+      { file: nil }
     end
 
     def self.expanding_parameters

lib/squib/deck.rb

@@ -131,5 +131,6 @@ module Squib
     require_relative 'dsl/text'
     require_relative 'dsl/triangle'
     require_relative 'dsl/units'
+    require_relative 'dsl/xlsx'
   end
 end

lib/squib/dsl/xlsx.rb

@@ -0,0 +1,41 @@
+require_relative '../args/input_file'
+require_relative '../args/import'
+require_relative '../import/xlsx_importer'
+require_relative '../errors_warnings/warn_unexpected_params'
+
+module Squib
+  # DSL method. See http://squib.readthedocs.io
+  def xlsx(opts = {}, &block)
+    DSL::Xlsx.new(__callee__).run(opts, &block)
+  end
+  module_function :xlsx
+
+  class Deck
+    # DSL method. See http://squib.readthedocs.io
+    def xlsx(opts = {}, &block)
+      DSL::Xlsx.new(__callee__).run(opts, &block)
+    end
+  end
+
+  module DSL
+    class Xlsx
+      include WarnUnexpectedParams
+      attr_reader :dsl_method, :block
+
+      def initialize(dsl_method)
+        @dsl_method = dsl_method
+      end
+
+      def self.accepted_params
+        %i( file sheet strip explode )
+      end
+
+      def run(opts,&block)
+        warn_if_unexpected opts
+        import_args = Args.extract_import opts
+        importer = Squib::Import::XlsxImporter.new
+        importer.import_to_dataframe(import_args, &block)
+      end
+    end
+  end
+end

lib/squib/import/quantity_exploder.rb

@@ -0,0 +1,18 @@
+module Squib
+  module Import
+    module QuantityExploder
+      def explode_quantities(data, qty)
+        return data unless data.col? qty.to_s.strip
+        qtys = data[qty]
+        new_data = Squib::DataFrame.new
+        data.each do |col, arr|
+          new_data[col] = []
+          qtys.each_with_index do |qty, index|
+            qty.to_i.times { new_data[col] << arr[index] }
+          end
+        end
+        return new_data
+      end
+    end
+  end
+end

lib/squib/import/xlsx_importer.rb

@@ -0,0 +1,27 @@
+require_relative 'quantity_exploder'
+
+module Squib::Import
+  class XlsxImporter
+    include Squib::Import::QuantityExploder
+    def import_to_dataframe(import, &block)
+      s = Roo::Excelx.new(import.file)
+      s.default_sheet = s.sheets[import.sheet]
+      data = Squib::DataFrame.new
+      s.first_column.upto(s.last_column) do |col|
+        header = s.cell(s.first_row, col).to_s
+        header.strip! if import.strip?
+        data[header] = []
+        (s.first_row + 1).upto(s.last_row) do |row|
+          cell = s.cell(row, col)
+          # Roo hack for avoiding unnecessary .0's on whole integers (https://github.com/roo-rb/roo/issues/139)
+          cell = s.excelx_value(row, col) if s.excelx_type(row, col) == [:numeric_or_formula, 'General']
+          cell.strip! if cell.respond_to?(:strip) && import.strip?
+          cell = block.yield(header, cell) unless block.nil?
+          data[header] << cell
+        end# row
+      end# col
+      explode_quantities(data, import.explode)
+    end
+  end
+end
+

spec/import/csv_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe Squib::Deck do
+  context '#csv' do
+    it 'loads basic csv data' do
+      expect(Squib.csv(file: csv_file('basic.csv')).to_h.to_h).to eq({
+        'h1' => [1, 3],
+        'h2' => [2, 4]
+        })
+    end
+
+    it 'collapses duplicate columns and warns' do
+      expect(Squib.logger).to receive(:warn)
+        .with('CSV duplicated the following column keys: h1,h1')
+      expect(Squib.csv(file: csv_file('dup_cols.csv')).to_h.to_h).to eq({
+        'h1' => [1, 3],
+        'h2' => [5, 7],
+        'H2' => [6, 8],
+        'h3' => [9, 10],
+        })
+    end
+
+    it 'strips spaces by default' do
+      expect(Squib.csv(file: csv_file('with_spaces.csv')).to_h).to eq({
+        'With Spaces' => ['a b c', 3],
+        'h2' => [2, 4],
+        'h3' => [3, nil]
+        })
+    end
+
+    it 'skips space stripping if told to' do
+      expect(Squib.csv(strip: false, file: csv_file('with_spaces.csv')).to_h).to eq({
+        '  With Spaces  ' => ['a b c      ', 3],
+        'h2' => [2, 4],
+        'h3' => [3, nil]
+        })
+    end
+
+    it 'explodes quantities' do
+      expect(Squib.csv(file: csv_file('qty.csv')).to_h).to eq({
+        'Name' => %w(Ha Ha Ha Ho),
+        'qty' => [3, 3, 3, 1],
+        })
+    end
+
+    it 'explodes quantities on specified header' do
+      expect(Squib.csv(explode: 'Quantity', file: csv_file('qty_named.csv')).to_h).to eq({
+        'Name' => %w(Ha Ha Ha Ho),
+        'Quantity' => [3, 3, 3, 1],
+        })
+    end
+
+    it 'loads inline data' do
+      hash = Squib.csv(data: "h1,h2\n1,2\n3,4")
+      expect(hash.to_h).to eq({
+        'h1' => [1, 3],
+        'h2' => [2, 4]
+        })
+    end
+
+    it 'loads csv with newlines' do
+      hash = Squib.csv(file: csv_file('newline.csv'))
+      expect(hash.to_h).to eq({
+        'title' => ['Foo'],
+        'level' => [1],
+        'notes' => ["a\nb"]
+      })
+    end
+
+    it 'loads custom CSV options' do
+      hash = Squib.csv(file: csv_file('custom_opts.csv'),
+                       col_sep: '-', quote_char: '|')
+      expect(hash.to_h).to eq({
+        'x' => ['p'],
+        'y' => ['q-r']
+        })
+    end
+
+    it 'yields to block when given' do
+      data = Squib.csv(file: csv_file('basic.csv')) do |header, value|
+        case header
+        when 'h1'
+          value * 2
+        else
+          'ha'
+        end
+      end
+      expect(data.to_h).to eq({
+        'h1' => [2, 6],
+        'h2' => %w(ha ha),
+        })
+    end
+
+    it 'replaces newlines whenever its a string' do
+      data = Squib.csv(file: csv_file('yield.csv')) do |header, value|
+        if value.respond_to? :gsub
+          value.gsub '%n', "\n"
+        else
+          value
+        end
+      end
+      expect(data.to_h).to eq({
+        'a' => ["foo\nbar", 1],
+        'b' => [1, "blah\n"],
+        })
+    end
+
+  end
+end

spec/import/xlsx_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+
+describe Squib::Deck do
+  context '#xlsx' do
+    it 'loads basic xlsx data' do
+      expect(Squib.xlsx(file: xlsx_file('basic.xlsx')).to_h).to eq({
+        'Name'           => %w(Larry Curly Mo),
+        'General Number' => %w(1 2 3), # general types always get loaded as strings with no conversion
+        'Actual Number' => [4.0, 5.0, 6.0], # numbers get auto-converted to integers
+        })
+    end
+
+    it 'loads xlsx with formulas' do
+      expect(Squib.xlsx(file: xlsx_file('formulas.xlsx')).to_h).to eq({
+        'A'   => %w(1 2),
+        'B'   => %w(3 4),
+        'Sum' => %w(4 6),
+        })
+    end
+
+    it 'loads xlsm files with macros' do
+      expect(Squib.xlsx(file: xlsx_file('with_macros.xlsm')).to_h).to eq({
+        'foo' => %w(8 10),
+        'bar' => %w(9 11),
+        })
+    end
+
+    it 'strips whitespace by default' do
+      expect(Squib.xlsx(file: xlsx_file('whitespace.xlsx')).to_h).to eq({
+          'With Whitespace' => ['foo', 'bar', 'baz'],
+        })
+    end
+
+    it 'does not strip whitespace when specified' do
+      expect(Squib.xlsx(file: xlsx_file('whitespace.xlsx'), strip: false).to_h).to eq({
+          '                  With Whitespace                  ' => ['foo             ', '      bar', '   baz  '],
+        })
+    end
+
+    it 'yields to block when given' do
+      data = Squib.xlsx(file: xlsx_file('basic.xlsx')) do |header, value|
+        case header
+        when 'Name'
+          'he'
+        when 'Actual Number'
+          value * 2
+        else
+          'ha'
+        end
+      end
+      expect(data.to_h).to eq({
+        'Name'           => %w(he he he),
+        'General Number' => %w(ha ha ha),
+        'Actual Number'  => [8.0, 10.0, 12.0],
+        })
+    end
+
+    it 'explodes quantities' do
+       expect(Squib.xlsx(explode: 'Quantity', file: xlsx_file('explode_quantities.xlsx')).to_h).to eq({
+        'Name' => ['Zergling', 'Zergling', 'Zergling', 'High Templar'],
+        'Quantity'  => %w(3 3 3 1),
+        })
+    end
+
+  end  
+end

spec/import/yaml_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe Squib::Deck do
+  context '#yaml' do
+    it 'loads basic data' do
+      expect(Squib.yaml(file: yaml_file('basic.yml')).to_h).to eq({
+        'Name'           => %w(Larry Curly Mo),
+        'Number' => [4.0, 5.0, 6.0], # numbers get auto-converted to integers
+        })
+    end
+
+    it 'explodes quantities' do
+     expect(Squib.yaml(explode: 'qty', file: yaml_file('qty.yml')).to_h).to eq({
+        'name' => %w(ha ha he),
+        'qty'  => [2, 2, 1],
+        })
+    end
+
+    it 'handles silence' do
+     expect(Squib.yaml(file: yaml_file('nilly.yml')).to_h).to eq({
+        'name' => %w(foo bar),
+        'desc'  => [nil, 'Hello'],
+        })
+    end
+
+    it 'yields to block when given' do
+      data = Squib.yaml(file: yaml_file('basic.yml')) do |header, value|
+        case header
+        when 'Name'
+          'he'
+        when 'Number'
+          value * 2
+        else
+          'ha'
+        end
+      end
+      expect(data.to_h).to eq({
+        'Name'   => %w(he he he),
+        'Number' => [8.0, 10.0, 12.0],
+        })
+    end
+  end
+end