diff --git a/.rubocop.yml b/.rubocop.yml index 46540ed..4f84157 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -3,7 +3,7 @@ AllCops: Documentation: Enabled: false Metrics/BlockLength: - ExcludedMethods: ['describe', 'helpers'] + ExcludedMethods: ['dataset_module', 'describe', 'helpers'] Metrics/AbcSize: Max: 20.45 Metrics/MethodLength: diff --git a/config/initializers/sequel.rb b/config/initializers/sequel.rb index e9010e5..69605bf 100644 --- a/config/initializers/sequel.rb +++ b/config/initializers/sequel.rb @@ -3,6 +3,8 @@ require 'pg' require 'sequel' +Sequel.extension :pg_json_ops Sequel.single_threaded = true Sequel.connect(ENV['DATABASE_URL'] || "postgres://localhost/frankfurter_#{App.env}") + .extension :pg_json diff --git a/db/migrate/002_denormalise_currencies.rb b/db/migrate/002_denormalise_currencies.rb new file mode 100644 index 0000000..3aeb767 --- /dev/null +++ b/db/migrate/002_denormalise_currencies.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +Sequel.migration do + up do + create_table :days do + date :date + jsonb :rates + index :date, unique: true + end + drop_table :currencies + end + + down do + create_table :currencies do + date :date + string :iso_code + float :rate + index %i[date iso_code], unique: true + end + drop_table :days + end +end diff --git a/lib/bank.rb b/lib/bank.rb index c6c2156..217f534 100644 --- a/lib/bank.rb +++ b/lib/bank.rb @@ -1,27 +1,41 @@ # frozen_string_literal: true -require 'currency' +require 'day' require 'bank/feed' module Bank - def self.fetch_all! - Currency.dataset.insert_conflict.multi_insert(Feed.historical.to_a) - end + class << self + def fetch_all! + data = Feed.historical.to_a + jsonify!(data) + Day.dataset.insert_conflict.multi_insert(data) + end - def self.fetch_ninety_days! - Currency.dataset.insert_conflict.multi_insert(Feed.ninety_days.to_a) - end + def fetch_ninety_days! + data = Feed.ninety_days.to_a + jsonify!(data) + Day.dataset.insert_conflict.multi_insert(data) + end - def self.fetch_current! - Currency.db.transaction do - Feed.current.each do |hsh| - Currency.find_or_create(hsh) + def fetch_current! + data = Feed.current.to_a + jsonify!(data) + Day.find_or_create(data.first) + end + + def replace_all! + data = Feed.historical.to_a + jsonify!(data) + Day.dataset.delete + Day.multi_insert(data) + end + + private + + def jsonify!(data) + data.each do |day| + day[:rates] = Sequel.pg_jsonb(day[:rates]) end end end - - def self.replace_all! - Currency.dataset.delete - Currency.multi_insert(Feed.historical.to_a) - end end diff --git a/lib/bank/feed.rb b/lib/bank/feed.rb index 59b90cb..4e05ea7 100644 --- a/lib/bank/feed.rb +++ b/lib/bank/feed.rb @@ -25,14 +25,10 @@ module Bank def each document.locate('gesmes:Envelope/Cube/Cube').each do |day| - date = Date.parse(day['time']) - day.locate('Cube').each do |record| - yield( - date: date, - iso_code: record['currency'], - rate: Float(record['rate']) - ) - end + yield(date: Date.parse(day['time']), + rates: day.nodes.each_with_object({}) do |currency, rates| + rates[currency[:currency]] = Float(currency[:rate]) + end) end end diff --git a/lib/currency.rb b/lib/currency.rb index 89853b9..7aa3dd1 100644 --- a/lib/currency.rb +++ b/lib/currency.rb @@ -1,30 +1,36 @@ # frozen_string_literal: true -class Currency < Sequel::Model +require 'day' +require 'forwardable' + +class Currency < Sequel::Model(Day.currencies) + class << self + extend Forwardable + + def_delegators :dataset, :latest, :between + end + dataset_module do - def latest(date = Date.today) - where(date: select(:date).where(Sequel.lit('date <= ?', date)) - .order(Sequel.desc(:date)) - .limit(1)) + def only(*iso_codes) + where(iso_code: iso_codes) end - def between(date_interval) - query = where(date: date_interval).order(:date) - length = date_interval.last - date_interval.first - if length > 365 - query.sampled('month') - elsif length > 90 - query.sampled('week') - else - query + def between(interval) + case interval.last - interval.first + when 0..90 then super + when 91..365 then super.sample('week') + else super.sample('month') end end - def sampled(precision) - sampled_date = Sequel.lit("date_trunc('#{precision}', date)") - select(:iso_code).select_append { avg(rate).as(rate) } - .select_append(sampled_date.as(:date)) - .group(:iso_code, sampled_date) + def sample(precision) + sampler = Sequel.function(:date_trunc, precision, :date) + + select(:iso_code) + .select_append { avg(rate).as(rate) } + .select_append(sampler.as(:date)) + .group(:iso_code, sampler) + .order(:date) end end end diff --git a/lib/day.rb b/lib/day.rb new file mode 100644 index 0000000..aa8de00 --- /dev/null +++ b/lib/day.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class Day < Sequel::Model + dataset_module do + def latest(date = Date.today) + where(date: select(:date).where(Sequel.lit('date <= ?', date)) + .order(Sequel.desc(:date)) + .limit(1)) + end + + def between(interval) + where(date: interval) + end + + def currencies + select(:date, + Sequel.lit('rates.key').as(:iso_code), + Sequel.lit('rates.value::text::float').as(:rate)) + .join(Sequel.function(:jsonb_each, :rates).lateral.as(:rates), true) + end + end +end diff --git a/lib/quote/end_of_day.rb b/lib/quote/end_of_day.rb index 6f838de..26ba460 100644 --- a/lib/quote/end_of_day.rb +++ b/lib/quote/end_of_day.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'currency' require 'quote/base' module Quote @@ -21,8 +20,10 @@ module Quote private def fetch_data + require 'currency' + scope = Currency.latest(date) - scope = scope.where(iso_code: symbols + [base]) if symbols + scope = scope.only(*(symbols + [base])) if symbols scope.naked end diff --git a/lib/quote/interval.rb b/lib/quote/interval.rb index e7a703e..74db894 100644 --- a/lib/quote/interval.rb +++ b/lib/quote/interval.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'currency' require 'quote/base' module Quote @@ -22,8 +21,10 @@ module Quote private def fetch_data + require 'currency' + scope = Currency.between(date) - scope = scope.where(iso_code: symbols + [base]) if symbols + scope = scope.only(*(symbols + [base])) if symbols scope.naked end diff --git a/spec/bank/feed_spec.rb b/spec/bank/feed_spec.rb index 5fa1bc3..84d2adb 100644 --- a/spec/bank/feed_spec.rb +++ b/spec/bank/feed_spec.rb @@ -15,35 +15,33 @@ module Bank it 'fetches current rates' do feed = Feed.current - feed.count.must_be :<, 40 + feed.count.must_be :==, 1 end it 'fetches rates for the past 90 days' do feed = Feed.ninety_days - feed.count.must_be :>, 33 * 60 + feed.count.must_be :>, 1 + feed.count.must_be :<=, 90 end it 'fetches historical rates' do feed = Feed.historical - feed.count.must_be :>, 33 * 3000 + feed.count.must_be :>, 90 end - it 'parses the date of a currency' do + it 'parses dates' do feed = Feed.current - currency = feed.first - currency[:date].must_be_kind_of Date + day = feed.first + day[:date].must_be_kind_of Date end - it 'parse the ISO code of a currency' do + it 'parses rates' do feed = Feed.current - currency = feed.first - currency[:iso_code].must_be_kind_of String - end - - it 'parses the rate of a currency' do - feed = Feed.current - currency = feed.first - currency[:rate].must_be_kind_of Float + day = feed.first + day[:rates].each do |iso_code, value| + iso_code.must_be_kind_of String + value.must_be_kind_of Float + end end end end diff --git a/spec/bank_spec.rb b/spec/bank_spec.rb index ffcd53b..0af21de 100644 --- a/spec/bank_spec.rb +++ b/spec/bank_spec.rb @@ -5,7 +5,7 @@ require 'bank' describe Bank do around do |test| - Currency.db.transaction do + Day.db.transaction do test.call raise Sequel::Rollback end @@ -20,39 +20,39 @@ describe Bank do end it 'fetches all rates' do - Currency.dataset.delete + Day.dataset.delete Bank.fetch_all! - Currency.count.must_be :positive? + Day.count.must_be :positive? end it 'skips existing records when fetching all rates' do - Currency.where { date < '2012-01-01' }.delete + Day.where { date < '2012-01-01' }.delete Bank.fetch_all! - Currency.where { date < '2012-01-01' }.count.must_be :positive? + Day.where { date < '2012-01-01' }.count.must_be :positive? end it 'fetches rates for last 90 days' do - Currency.dataset.delete + Day.dataset.delete Bank.fetch_ninety_days! - Currency.count.must_be :positive? + Day.count.must_be :positive? end it 'skips existing records when fetching rates for last 90 days' do cutoff = Date.today - 60 - Currency.where { date < cutoff }.delete + Day.where { date < cutoff }.delete Bank.fetch_ninety_days! - Currency.where { date < cutoff }.count.must_be :positive? + Day.where { date < cutoff }.count.must_be :positive? end it 'fetches current rates' do - Currency.dataset.delete + Day.dataset.delete Bank.fetch_current! - Currency.count.must_be :positive? + Day.count.must_be :positive? end it 'replaces all rates' do - Currency.dataset.delete + Day.dataset.delete Bank.fetch_all! - Currency.count.must_be :positive? + Day.count.must_be :positive? end end diff --git a/spec/currency_spec.rb b/spec/currency_spec.rb index 6c5a3c9..a8da87c 100644 --- a/spec/currency_spec.rb +++ b/spec/currency_spec.rb @@ -4,52 +4,49 @@ require_relative 'helper' require 'currency' describe Currency do - describe '.current' do - it 'returns current rates' do - date = Currency.order(Sequel.desc(:date)).first.date - rates = Currency.latest - rates.first.date.must_equal date - end - - it 'returns latest rates before given date' do - date = Date.parse('2010-01-01') - rates = Currency.latest(date) - rates.first.date.must_be :<=, date - end - - it 'returns nothing if there are no rates before given date' do - rates = Currency.latest(Date.parse('1998-01-01')) - rates.must_be_empty + describe '.latest' do + it 'returns latest rates' do + data = Currency.latest.all + data.count.must_be :>, 1 end end describe '.between' do - it 'returns rates between given dates' do - start_date = Date.parse('2010-01-01') - end_date = Date.parse('2010-01-31') - dates = Currency.between((start_date..end_date)).map(:date).uniq.sort - dates.first.must_be :>=, start_date - dates.last.must_be :<=, end_date + let(:day) do + Date.parse('2010-01-01') end - it 'returns nothing if there are no rates between given dates' do - date_interval = (Date.parse('1998-01-01')..Date.parse('1998-01-31')) - Currency.between(date_interval).must_be_empty + it 'returns everything up to 90 days' do + interval = day..day + 90 + Currency.between(interval).map(:date).uniq.count.must_be :>, 30 end - it 'returns all rates up to 90 days' do - date_interval = (Date.parse('2010-01-01')..Date.parse('2010-03-01')) - Currency.between(date_interval).map(:date).uniq.count.must_be :>, 30 + it 'samples weekly over 90 but below 366 days' do + interval = day..day + 365 + Currency.between(interval).map(:date).uniq.count.must_be :<=, 52 end - it 'samples weeks over 90 days and below 366 days' do - date_interval = (Date.parse('2010-01-01')..Date.parse('2010-12-31')) - Currency.between(date_interval).map(:date).uniq.count.must_be :<=, 52 + it 'samples monthly over 365 days' do + interval = day..day + 400 + Currency.between(interval).map(:date).uniq.count.must_be :<=, 120 end - it 'samples months over 365 days' do - date_interval = (Date.parse('2001-01-01')..Date.parse('2010-12-31')) - Currency.between(date_interval).map(:date).uniq.count.must_be :<=, 120 + it 'sorts by date' do + interval = day..day + 100 + dates = Currency.between(interval).map(:date) + dates.must_equal dates.sort + end + end + + describe '.only' do + it 'filters symbols' do + iso_codes = %w[CAD USD] + data = Currency.latest.only(*iso_codes).all + data.map(&:iso_code).sort.must_equal iso_codes + end + + it 'returns nothing if no matches' do + Currency.only('FOO').all.must_be_empty end end end diff --git a/spec/day_spec.rb b/spec/day_spec.rb new file mode 100644 index 0000000..9decdb7 --- /dev/null +++ b/spec/day_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require_relative 'helper' +require 'day' + +describe Day do + describe '.latest' do + it 'returns latest rates before given date' do + date = Date.parse('2010-01-01') + data = Day.latest(date) + data.first.date.must_be :<=, date + end + + it 'returns nothing if there are no rates before given date' do + Day.latest(Date.parse('1998-01-01')).must_be_empty + end + end + + describe '.between' do + it 'returns rates between given dates' do + start_date = Date.parse('2010-01-01') + end_date = Date.parse('2010-01-31') + dates = Day.between((start_date..end_date)).map(:date).sort + dates.first.must_be :>=, start_date + dates.last.must_be :<=, end_date + end + + it 'returns nothing if there are no rates between given dates' do + interval = (Date.parse('1998-01-01')..Date.parse('1998-01-31')) + Day.between(interval).must_be_empty + end + end +end diff --git a/spec/edge_cases_spec.rb b/spec/edge_cases_spec.rb index 42ccb92..4962ea8 100644 --- a/spec/edge_cases_spec.rb +++ b/spec/edge_cases_spec.rb @@ -4,7 +4,7 @@ require_relative 'helper' require 'rack/test' require 'web/server' -describe 'the API' do +describe 'the server' do include Rack::Test::Methods let(:app) { Sinatra::Application } @@ -39,10 +39,10 @@ describe 'the API' do end it 'does not return stale dates' do - Currency.db.transaction do + Day.db.transaction do get '/latest' date = json['date'] - Currency.latest.delete + Day.latest.delete get '/latest' json['date'].wont_equal date raise Sequel::Rollback