Add Excel to preprocessing excel file

Gemfile.lock

@@ -4,6 +4,8 @@ PATH
     tallty_import_export (0.1.0)
       activesupport
       caxlsx
+      redis
+      redis-objects
       roo
       roo-xls
       tallty_form
@@ -36,6 +38,9 @@ GEM
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     rake (12.3.3)
+    redis (4.2.5)
+    redis-objects (1.5.0)
+      redis (~> 4.0)
     roo (2.8.3)
       nokogiri (~> 1)
       rubyzip (>= 1.3.0, < 3.0.0)

lib/tallty_import_export.rb

 require "active_support/core_ext/hash"
 require "tallty_import_export/version"
 require 'axlsx'
+require 'redis'
 require 'tallty_form'
 
 module TalltyImportExport
@@ -13,6 +14,15 @@ module TalltyImportExport
   autoload :Import
   autoload :Context
   autoload :FormConvert
+  autoload :Excel
 
   class Error < StandardError; end
+
+  def self.redis
+    @redis ||= Redis.new
+  end
+
+  def self.redis=(redis)
+    @redis = redis
+  end
 end

lib/tallty_import_export/excel.rb

+module TalltyImportExport
+  class Excel
+    require 'redis-objects'
+    require 'roo'
+
+    attr_reader :uid, :cache
+
+    def initialize uid = SecureRandom.hex(8)
+      @uid = uid
+      @cache = ::Redis::HashKey.new(@uid, expiration: 60 * 60 * 1, marshal: true)
+    end
+
+    def load file, **opts
+      opts = opts.with_indifferent_access
+
+      file_path = file.is_a?(String) ? file : file.path
+      book = ::Roo::Spreadsheet.open(file_path, extension: File.extname(file_path) == '.xls' ? :xls : :xlsx)
+
+      read_title_from_row = opts[:read_title_from_row] || 1
+      read_data_from_row = opts[:read_data_from_row] || 2
+      titles = book.row(read_title_from_row)
+      rows = []
+
+      read_data_from_row.upto(book.last_row) do |i|
+        row = book.row(i)[0 .. titles.count]
+        out = {}
+        titles.each_with_index do |title, index|
+          out[title] = row[index]
+        end
+        rows << out
+      end
+      cache['uid'] = uid
+      cache['rows'] = rows
+      cache['titles'] = titles
+    end
+
+    def rows
+      Rows.new(cache['rows'])
+    end
+
+    def titles
+      cache['titles']
+    end
+
+    def data
+      cache.all
+    end
+
+    class Rows < Array
+      # excel_hash { key => excel name }
+      def each_with_excel_hash excel_hash
+        mapping = excel_hash.invert
+        each do |row|
+          line_info = row.reduce({}) do |out, (k, v)|
+            out[mapping[k]] = v
+            out
+          end
+          yield line_info
+        end
+      end
+    end
+  end
+end

lib/tallty_import_export/import.rb

@@ -15,7 +15,8 @@ module TalltyImportExport
     # convert: 导入时候，把excel的内容转换成导入时候代码逻辑需要的内容
     # primary_key: 是否是主键
 
-    def import_xlsx xlsx_file, associations, options={}
+    # xlsx_file 为 file path or file object or TalltyImportExport::Excel.new
+    def import_xlsx xlsx_file, associations, **options
       # 先处理获取出来Excel每行的数据, line_info
       options = options.with_indifferent_access
 
@@ -26,12 +27,24 @@ module TalltyImportExport
         h[header[:key]] = header[:name]
         h
       end
+
+      if TalltyImportExport::Excel === xlsx_file
+        xlsx_file.rows.each_with_excel_hash(excel_hash) do |line_info|
+          process_line_info(line_info, associations)
+        end
+      else
         file_path = xlsx_file.is_a?(String) ? xlsx_file : xlsx_file.path
         xlsx = ::Roo::Excelx.new(file_path)
         xlsx.each_with_pagename do |_sheetname, sheet|
           sheet.each(**excel_hash).with_index do |line_info, index|
             next if index == 0
+            process_line_info(line_info, associations)
+          end
+        end
+      end
+    end
 
+    def process_line_info line_info, associations
       # 转换处理导入的数据格式
       line_info = convert_data(line_info)
 
@@ -39,8 +52,6 @@ module TalltyImportExport
       import_record(line_info, associations)
       context.last_line_info = line_info
     end
-      end
-    end
 
     def convert_data line_info
       line_info.with_indifferent_access.reduce({}) do |h, (k, v)|

spec/excel_spec.rb

+RSpec.describe TalltyImportExport::Excel do
+  it do
+    @excel = TalltyImportExport::Excel.new
+    expect(@excel.uid).not_to be_nil
+    @excel.load('./spec/test1.xlsx')
+    expect(@excel.titles).to eq(["名称", "学号", "副学号", "meta字段1", "meta字段2", "累加值"])
+    expect(@excel.rows[0]).to eq({"名称"=>1, "学号"=>20070101, "副学号"=>1, "meta字段1"=>"meta1", "meta字段2"=>"metaA", "累加值"=>1})
+  end
+end

spec/importable_spec.rb

@@ -32,7 +32,7 @@ RSpec.describe TalltyImportExport::Importable do
     end
   end
 
-  it "has a version number" do
+  it 'import file directly' do
     # 名称	学号	  副学号	meta字段1	meta字段2
     # 1	  20070101	1	    meta1	    metaA
     # 2	  20070102	2	    meta2	    metaB
@@ -50,4 +50,27 @@ RSpec.describe TalltyImportExport::Importable do
       {"name"=>5, "code"=>"20070105_5", "sub_code"=>nil, "meta"=>{"meta1"=>"meta5", "meta2"=>"metaE" }, "sum" => 15 },
     ])
   end
+
+  it 'import excel object' do
+    # 名称	学号	  副学号	meta字段1	meta字段2
+    # 1	  20070101	1	    meta1	    metaA
+    # 2	  20070102	2	    meta2	    metaB
+    # 3	  20070103	3	    meta3	    metaC
+    # 4	  20070104	4	    meta4	    metaD
+    # 5	  20070105	5	    meta5	    metaE
+    @result = []
+
+    @excel = TalltyImportExport::Excel.new
+    @excel.load('./spec/test1.xlsx')
+
+    Test.import_xlsx(@excel, @result)
+
+    expect(@result).to eq([
+      {"name"=>1, "code"=>"20070101_1", "sub_code"=>nil, "meta"=>{"meta1"=>"meta1", "meta2"=>"metaA" }, "sum" => 1 },
+      {"name"=>2, "code"=>"20070102_2", "sub_code"=>nil, "meta"=>{"meta1"=>"meta2", "meta2"=>"metaB" }, "sum" => 3 },
+      {"name"=>3, "code"=>"20070103_3", "sub_code"=>nil, "meta"=>{"meta1"=>"meta3", "meta2"=>"metaC" }, "sum" => 6 },
+      {"name"=>4, "code"=>"20070104_4", "sub_code"=>nil, "meta"=>{"meta1"=>"meta4", "meta2"=>"metaD" }, "sum" => 10 },
+      {"name"=>5, "code"=>"20070105_5", "sub_code"=>nil, "meta"=>{"meta1"=>"meta5", "meta2"=>"metaE" }, "sum" => 15 },
+    ])
+  end
 end

spec/tallty_import_export_spec.rb

 RSpec.describe TalltyImportExport do
-  it "has a version number" do
+  it 'has a version number' do
     expect(TalltyImportExport::VERSION).not_to be nil
   end
+
+  it 'TalltyImportExport.redis return a redis obj' do
+    expect(TalltyImportExport.redis.class).to eq(Redis)
+  end
 end

tallty_import_export.gemspec

@@ -29,4 +29,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency('roo-xls')
   spec.add_dependency('tallty_form')
   spec.add_dependency('activesupport')
+  spec.add_dependency('redis')
+  spec.add_dependency('redis-objects')
 end