Denormalise database schema

This way, we will have no issue fitting the entire dataset on the free tier of Heroku

.rubocop.yml

@@ -3,7 +3,7 @@ AllCops:
 Documentation:
   Enabled: false
 Metrics/BlockLength:
-  ExcludedMethods: ['describe', 'helpers']
+  ExcludedMethods: ['dataset_module', 'describe', 'helpers']
 Metrics/AbcSize:
   Max: 20.45
 Metrics/MethodLength:

config/initializers/sequel.rb

@@ -3,6 +3,8 @@
 require 'pg'
 require 'sequel'
 
+Sequel.extension :pg_json_ops
 Sequel.single_threaded = true
 Sequel.connect(ENV['DATABASE_URL'] ||
                "postgres://localhost/frankfurter_#{App.env}")
+      .extension :pg_json

db/migrate/002_denormalise_currencies.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table :days do
+      date  :date
+      jsonb :rates
+      index :date, unique: true
+    end
+    drop_table :currencies
+  end
+
+  down do
+    create_table :currencies do
+      date    :date
+      string  :iso_code
+      float   :rate
+      index %i[date iso_code], unique: true
+    end
+    drop_table :days
+  end
+end

lib/bank.rb

@@ -1,27 +1,41 @@
 # frozen_string_literal: true
 
-require 'currency'
+require 'day'
 require 'bank/feed'
 
 module Bank
-  def self.fetch_all!
-    Currency.dataset.insert_conflict.multi_insert(Feed.historical.to_a)
+  class << self
+    def fetch_all!
+      data = Feed.historical.to_a
+      jsonify!(data)
+      Day.dataset.insert_conflict.multi_insert(data)
     end
 
-  def self.fetch_ninety_days!
-    Currency.dataset.insert_conflict.multi_insert(Feed.ninety_days.to_a)
+    def fetch_ninety_days!
+      data = Feed.ninety_days.to_a
+      jsonify!(data)
+      Day.dataset.insert_conflict.multi_insert(data)
     end
 
-  def self.fetch_current!
-    Currency.db.transaction do
-      Feed.current.each do |hsh|
-        Currency.find_or_create(hsh)
-      end
-    end
+    def fetch_current!
+      data = Feed.current.to_a
+      jsonify!(data)
+      Day.find_or_create(data.first)
     end
 
-  def self.replace_all!
-    Currency.dataset.delete
-    Currency.multi_insert(Feed.historical.to_a)
+    def replace_all!
+      data = Feed.historical.to_a
+      jsonify!(data)
+      Day.dataset.delete
+      Day.multi_insert(data)
+    end
+
+    private
+
+    def jsonify!(data)
+      data.each do |day|
+        day[:rates] = Sequel.pg_jsonb(day[:rates])
+      end
+    end
   end
 end

lib/bank/feed.rb

@@ -25,14 +25,10 @@ module Bank
 
     def each
       document.locate('gesmes:Envelope/Cube/Cube').each do |day|
-        date = Date.parse(day['time'])
-        day.locate('Cube').each do |record|
-          yield(
-            date: date,
-            iso_code: record['currency'],
-            rate: Float(record['rate'])
-          )
-        end
+        yield(date: Date.parse(day['time']),
+              rates: day.nodes.each_with_object({}) do |currency, rates|
+                rates[currency[:currency]] = Float(currency[:rate])
+              end)
       end
     end
 

lib/currency.rb

@@ -1,30 +1,36 @@
 # frozen_string_literal: true
 
-class Currency < Sequel::Model
+require 'day'
+require 'forwardable'
+
+class Currency < Sequel::Model(Day.currencies)
+  class << self
+    extend Forwardable
+
+    def_delegators :dataset, :latest, :between
+  end
+
   dataset_module do
-    def latest(date = Date.today)
-      where(date: select(:date).where(Sequel.lit('date <= ?', date))
-                               .order(Sequel.desc(:date))
-                               .limit(1))
+    def only(*iso_codes)
+      where(iso_code: iso_codes)
     end
 
-    def between(date_interval)
-      query = where(date: date_interval).order(:date)
-      length = date_interval.last - date_interval.first
-      if length > 365
-        query.sampled('month')
-      elsif length > 90
-        query.sampled('week')
-      else
-        query
+    def between(interval)
+      case interval.last - interval.first
+      when 0..90 then super
+      when 91..365 then super.sample('week')
+      else super.sample('month')
       end
     end
 
-    def sampled(precision)
-      sampled_date = Sequel.lit("date_trunc('#{precision}', date)")
-      select(:iso_code).select_append { avg(rate).as(rate) }
-                       .select_append(sampled_date.as(:date))
-                       .group(:iso_code, sampled_date)
+    def sample(precision)
+      sampler = Sequel.function(:date_trunc, precision, :date)
+
+      select(:iso_code)
+        .select_append { avg(rate).as(rate) }
+        .select_append(sampler.as(:date))
+        .group(:iso_code, sampler)
+        .order(:date)
     end
   end
 end

lib/day.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class Day < Sequel::Model
+  dataset_module do
+    def latest(date = Date.today)
+      where(date: select(:date).where(Sequel.lit('date <= ?', date))
+                               .order(Sequel.desc(:date))
+                               .limit(1))
+    end
+
+    def between(interval)
+      where(date: interval)
+    end
+
+    def currencies
+      select(:date,
+             Sequel.lit('rates.key').as(:iso_code),
+             Sequel.lit('rates.value::text::float').as(:rate))
+        .join(Sequel.function(:jsonb_each, :rates).lateral.as(:rates), true)
+    end
+  end
+end

lib/quote/end_of_day.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'currency'
 require 'quote/base'
 
 module Quote
@@ -21,8 +20,10 @@ module Quote
     private
 
     def fetch_data
+      require 'currency'
+
       scope = Currency.latest(date)
-      scope = scope.where(iso_code: symbols + [base]) if symbols
+      scope = scope.only(*(symbols + [base])) if symbols
 
       scope.naked
     end

lib/quote/interval.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'currency'
 require 'quote/base'
 
 module Quote
@@ -22,8 +21,10 @@ module Quote
     private
 
     def fetch_data
+      require 'currency'
+
       scope = Currency.between(date)
-      scope = scope.where(iso_code: symbols + [base]) if symbols
+      scope = scope.only(*(symbols + [base])) if symbols
 
       scope.naked
     end

spec/bank/feed_spec.rb

@@ -15,35 +15,33 @@ module Bank
 
     it 'fetches current rates' do
       feed = Feed.current
-      feed.count.must_be :<, 40
+      feed.count.must_be :==, 1
     end
 
     it 'fetches rates for the past 90 days' do
       feed = Feed.ninety_days
-      feed.count.must_be :>, 33 * 60
+      feed.count.must_be :>, 1
+      feed.count.must_be :<=, 90
     end
 
     it 'fetches historical rates' do
       feed = Feed.historical
-      feed.count.must_be :>, 33 * 3000
+      feed.count.must_be :>, 90
     end
 
-    it 'parses the date of a currency' do
+    it 'parses dates' do
       feed = Feed.current
-      currency = feed.first
-      currency[:date].must_be_kind_of Date
+      day = feed.first
+      day[:date].must_be_kind_of Date
     end
 
-    it 'parse the ISO code of a currency' do
+    it 'parses rates' do
       feed = Feed.current
-      currency = feed.first
-      currency[:iso_code].must_be_kind_of String
-    end
-
-    it 'parses the rate of a currency' do
-      feed = Feed.current
-      currency = feed.first
-      currency[:rate].must_be_kind_of Float
+      day = feed.first
+      day[:rates].each do |iso_code, value|
+        iso_code.must_be_kind_of String
+        value.must_be_kind_of Float
+      end
     end
   end
 end

spec/bank_spec.rb

@@ -5,7 +5,7 @@ require 'bank'
 
 describe Bank do
   around do |test|
-    Currency.db.transaction do
+    Day.db.transaction do
       test.call
       raise Sequel::Rollback
     end
@@ -20,39 +20,39 @@ describe Bank do
   end
 
   it 'fetches all rates' do
-    Currency.dataset.delete
+    Day.dataset.delete
     Bank.fetch_all!
-    Currency.count.must_be :positive?
+    Day.count.must_be :positive?
   end
 
   it 'skips existing records when fetching all rates' do
-    Currency.where { date < '2012-01-01' }.delete
+    Day.where { date < '2012-01-01' }.delete
     Bank.fetch_all!
-    Currency.where { date < '2012-01-01' }.count.must_be :positive?
+    Day.where { date < '2012-01-01' }.count.must_be :positive?
   end
 
   it 'fetches rates for last 90 days' do
-    Currency.dataset.delete
+    Day.dataset.delete
     Bank.fetch_ninety_days!
-    Currency.count.must_be :positive?
+    Day.count.must_be :positive?
   end
 
   it 'skips existing records when fetching rates for last 90 days' do
     cutoff = Date.today - 60
-    Currency.where { date < cutoff }.delete
+    Day.where { date < cutoff }.delete
     Bank.fetch_ninety_days!
-    Currency.where { date < cutoff }.count.must_be :positive?
+    Day.where { date < cutoff }.count.must_be :positive?
   end
 
   it 'fetches current rates' do
-    Currency.dataset.delete
+    Day.dataset.delete
     Bank.fetch_current!
-    Currency.count.must_be :positive?
+    Day.count.must_be :positive?
   end
 
   it 'replaces all rates' do
-    Currency.dataset.delete
+    Day.dataset.delete
     Bank.fetch_all!
-    Currency.count.must_be :positive?
+    Day.count.must_be :positive?
   end
 end

spec/currency_spec.rb

@@ -4,52 +4,49 @@ require_relative 'helper'
 require 'currency'
 
 describe Currency do
-  describe '.current' do
-    it 'returns current rates' do
-      date = Currency.order(Sequel.desc(:date)).first.date
-      rates = Currency.latest
-      rates.first.date.must_equal date
-    end
-
-    it 'returns latest rates before given date' do
-      date = Date.parse('2010-01-01')
-      rates = Currency.latest(date)
-      rates.first.date.must_be :<=, date
-    end
-
-    it 'returns nothing if there are no rates before given date' do
-      rates = Currency.latest(Date.parse('1998-01-01'))
-      rates.must_be_empty
+  describe '.latest' do
+    it 'returns latest rates' do
+      data = Currency.latest.all
+      data.count.must_be :>, 1
     end
   end
 
   describe '.between' do
-    it 'returns rates between given dates' do
-      start_date = Date.parse('2010-01-01')
-      end_date = Date.parse('2010-01-31')
-      dates = Currency.between((start_date..end_date)).map(:date).uniq.sort
-      dates.first.must_be :>=, start_date
-      dates.last.must_be :<=, end_date
+    let(:day) do
+      Date.parse('2010-01-01')
     end
 
-    it 'returns nothing if there are no rates between given dates' do
-      date_interval = (Date.parse('1998-01-01')..Date.parse('1998-01-31'))
-      Currency.between(date_interval).must_be_empty
+    it 'returns everything up to 90 days' do
+      interval = day..day + 90
+      Currency.between(interval).map(:date).uniq.count.must_be :>, 30
     end
 
-    it 'returns all rates up to 90 days' do
-      date_interval = (Date.parse('2010-01-01')..Date.parse('2010-03-01'))
-      Currency.between(date_interval).map(:date).uniq.count.must_be :>, 30
+    it 'samples weekly over 90 but below 366 days' do
+      interval = day..day + 365
+      Currency.between(interval).map(:date).uniq.count.must_be :<=, 52
     end
 
-    it 'samples weeks over 90 days and below 366 days' do
-      date_interval = (Date.parse('2010-01-01')..Date.parse('2010-12-31'))
-      Currency.between(date_interval).map(:date).uniq.count.must_be :<=, 52
+    it 'samples monthly over 365 days' do
+      interval = day..day + 400
+      Currency.between(interval).map(:date).uniq.count.must_be :<=, 120
     end
 
-    it 'samples months over 365 days' do
-      date_interval = (Date.parse('2001-01-01')..Date.parse('2010-12-31'))
-      Currency.between(date_interval).map(:date).uniq.count.must_be :<=, 120
+    it 'sorts by date' do
+      interval = day..day + 100
+      dates = Currency.between(interval).map(:date)
+      dates.must_equal dates.sort
+    end
+  end
+
+  describe '.only' do
+    it 'filters symbols' do
+      iso_codes = %w[CAD USD]
+      data = Currency.latest.only(*iso_codes).all
+      data.map(&:iso_code).sort.must_equal iso_codes
+    end
+
+    it 'returns nothing if no matches' do
+      Currency.only('FOO').all.must_be_empty
     end
   end
 end

spec/day_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require 'day'
+
+describe Day do
+  describe '.latest' do
+    it 'returns latest rates before given date' do
+      date = Date.parse('2010-01-01')
+      data = Day.latest(date)
+      data.first.date.must_be :<=, date
+    end
+
+    it 'returns nothing if there are no rates before given date' do
+      Day.latest(Date.parse('1998-01-01')).must_be_empty
+    end
+  end
+
+  describe '.between' do
+    it 'returns rates between given dates' do
+      start_date = Date.parse('2010-01-01')
+      end_date = Date.parse('2010-01-31')
+      dates = Day.between((start_date..end_date)).map(:date).sort
+      dates.first.must_be :>=, start_date
+      dates.last.must_be :<=, end_date
+    end
+
+    it 'returns nothing if there are no rates between given dates' do
+      interval = (Date.parse('1998-01-01')..Date.parse('1998-01-31'))
+      Day.between(interval).must_be_empty
+    end
+  end
+end

spec/edge_cases_spec.rb

@@ -4,7 +4,7 @@ require_relative 'helper'
 require 'rack/test'
 require 'web/server'
 
-describe 'the API' do
+describe 'the server' do
   include Rack::Test::Methods
 
   let(:app) { Sinatra::Application }
@@ -39,10 +39,10 @@ describe 'the API' do
   end
 
   it 'does not return stale dates' do
-    Currency.db.transaction do
+    Day.db.transaction do
       get '/latest'
       date = json['date']
-      Currency.latest.delete
+      Day.latest.delete
       get '/latest'
       json['date'].wont_equal date
       raise Sequel::Rollback