feat: expected statements

Gemfile

@@ -5,8 +5,8 @@ source "https://rubygems.org"
 ruby "3.3.2"
 
 gem "activesupport"
-gem "camt_parser", git: "https://github.com/railslove/camt_parser.git"
-gem "cmxl", git: "https://github.com/railslove/cmxl"
+gem "sepa_file_parser"
+gem "cmxl"
 gem "epics"
 gem "faraday"
 gem "grape"

Gemfile.lock

@@ -1,17 +1,3 @@
-GIT
-  remote: https://github.com/railslove/camt_parser.git
-  revision: 3996d9dce6c4dbdc950c6d607102e3c4ce52ca52
-  specs:
-    camt_parser (1.0.2)
-      nokogiri
-
-GIT
-  remote: https://github.com/railslove/cmxl
-  revision: b92aea3de958426119d76d043a886233a313f43f
-  specs:
-    cmxl (1.5.0)
-      rchardet
-
 GIT
   remote: https://github.com/railslove/king_dtaus_ruby_3.git
   revision: 28b077aa6b2cf5590e322d0d207236adccf2d543
@@ -45,6 +31,8 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     builder (3.3.0)
+    cmxl (2.2)
+      rchardet
     coderay (1.1.3)
     concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
@@ -55,6 +43,7 @@ GEM
     database_cleaner-sequel (2.0.2)
       database_cleaner-core (~> 2.0.0)
       sequel
+    date (3.4.1)
     diff-lcs (1.5.1)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
@@ -123,7 +112,7 @@ GEM
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2024.0604)
-    mini_portile2 (2.8.7)
+    mini_portile2 (2.8.8)
     minitest (5.15.0)
     multi_json (1.15.0)
     mustermann (2.0.2)
@@ -134,7 +123,7 @@ GEM
       uri
     netrc (0.11.0)
     nio4r (2.7.4)
-    nokogiri (1.16.6)
+    nokogiri (1.18.2)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     parallel (1.25.1)
@@ -149,7 +138,7 @@ GEM
     puma (6.4.3)
       nio4r (~> 2.0)
     raabro (1.4.0)
-    racc (1.8.0)
+    racc (1.8.1)
     rack (2.2.9)
     rack-accept (0.4.5)
       rack (>= 0.4)
@@ -160,7 +149,7 @@ GEM
       rack (>= 1.0, < 3)
     rainbow (3.1.1)
     rake (13.2.1)
-    rchardet (1.8.0)
+    rchardet (1.9.0)
     redis (4.8.1)
     regexp_parser (2.9.2)
     rest-client (2.1.0)
@@ -210,6 +199,10 @@ GEM
     sentry-sidekiq (5.18.0)
       sentry-ruby (~> 5.18.0)
       sidekiq (>= 3.0)
+    sepa_file_parser (0.4.0)
+      bigdecimal
+      nokogiri
+      time
     sepa_king (0.12.0)
       activemodel (>= 3.1)
       iban-tools
@@ -239,6 +232,8 @@ GEM
       rubocop-performance (~> 1.21.0)
     statsd-ruby (1.5.0)
     tilt (2.4.0)
+    time (0.4.1)
+      date
     timecop (0.9.10)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
@@ -260,8 +255,7 @@ DEPENDENCIES
   activesupport
   airborne
   barnes
-  camt_parser!
-  cmxl!
+  cmxl
   database_cleaner-sequel
   dotenv
   epics
@@ -287,6 +281,7 @@ DEPENDENCIES
   rubocop
   sentry-ruby
   sentry-sidekiq
+  sepa_file_parser
   sepa_king
   sequel
   sidekiq

box/apis/v2/documentation.yml

@@ -84,7 +84,7 @@ Statement Created:
   account_id,
   statement: {
     id, account (iban), name, bic, iban, type, amount (cents), date,
-    remittance_information
+    remittance_information, expected, reversal
   }
 }
 ```

box/business_processes/import_bank_statement.rb

@@ -1,14 +1,10 @@
 # frozen_string_literal: true
 
 require "cmxl"
-
 require_relative "../models/account"
 require_relative "../models/bank_statement"
 require_relative "../../lib/checksum_generator"
 
-# more general matching regex that covers both newlines and newlines with dashes
-Cmxl.config[:statement_separator] = /(\n-?)(?=:20)/m
-
 module Box
   module BusinessProcesses
     class ImportBankStatement

box/business_processes/import_statements.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require "cmxl"
 require "camt_parser"
+require "cmxl"
 
 require_relative "../models/account"
 require_relative "../models/bank_statement"
@@ -113,8 +113,10 @@ def self.statement_attributes_from_bank_transaction(transaction, bank_statement)
           eref: transaction.respond_to?(:eref) ? transaction.eref : transaction.sepa["EREF"],
           mref: transaction.respond_to?(:mref) ? transaction.mref : transaction.sepa["MREF"],
           svwz: transaction.respond_to?(:svwz) ? transaction.svwz : transaction.sepa["SVWZ"],
-          tx_id: transaction.try(:primanota) || transaction.try(:transaction_id),
-          creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"]
+          tx_id: transaction.try(:transaction_id),
+          creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"],
+          expected: transaction.try(:expected?),
+          reversal: transaction..try(:reversal?)
         }
       end
     end

box/entities/statement.rb

@@ -12,6 +12,8 @@ class Statement < Grape::Entity
       expose :bic
       expose :iban
       expose :type, documentation: {type: "Enum", desc: "Type of statement", values: %w[credit debit]}
+      expose :expected, documentation: {type: "Boolean", desc: "Expected statement are not yet confirmed"}
+      expose :reversal, documentation: {type: "Boolean", desc: "Reversal of a previous transaction"}
       expose :amount, documentation: {type: "Integer", desc: "Amount in cents"}
       expose :date
       expose(:remittance_information, documentation: {type: "String", desc: "Wire transfer reference"}) { |statement| statement[:svwz] || statement[:information] }

box/entities/v2/transaction.rb

@@ -12,6 +12,8 @@ class Transaction < Grape::Entity
         expose :iban
         expose :bic
         expose :type
+        expose :expected
+        expose :reversal
         expose :amount, as: "amount_in_cents"
         expose :date, as: "executed_on"
         expose(:settled_at) { |trx| trx.settled ? trx.date : nil }

box/jobs/fetch_statements.rb

@@ -2,7 +2,7 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
+require "sepa_file_parser"
 require "cmxl"
 require "epics"
 require "sequel"
@@ -84,7 +84,7 @@ def camt53(client, from, to)
         combined_camt = client.C53(from.to_s(:db), to.to_s(:db))
         return unless combined_camt.any?
 
-        combined_camt.map { |chunk| CamtParser::String.parse(chunk).statements }.flatten
+        combined_camt.map { |chunk| SepaFileParser::String.parse(chunk).statements }.flatten
       end
 
       def mt940(client, from, to)

box/jobs/fetch_upcoming_statements.rb

@@ -2,8 +2,6 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 

box/jobs/queue_fetch_statements.rb

@@ -2,8 +2,6 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 

box/jobs/queue_fetch_upcoming_statements.rb

@@ -2,8 +2,6 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 

box/models/statement.rb

@@ -84,6 +84,14 @@ def debit?
       debit
     end
 
+    def reversal?
+      reversal
+    end
+
+    def expected?
+      !!expected
+    end
+
     def type
       debit? ? "debit" : "credit"
     end

db/migrations/20250121163300_add_meta_data_to_statement.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    add_column :statements, :expected, :boolean
+    add_column :statements, :reversal, :boolean
+  end
+end

db/schema.rb

@@ -117,6 +117,8 @@
       column :settled, "boolean", :default=>true
       column :sha, "text"
       column :tx_id, "text"
+      column :expected, "boolean"
+      column :reversal, "boolean"
 
       index [:sha], :name=>:statements_sha2_index, :unique=>true
       index [:sha_bak], :name=>:statements_sha_key, :unique=>true

spec/apis/v2/transactions_spec.rb

@@ -16,6 +16,8 @@ module Box
         amount_in_cents: :integer,
         executed_on: :date,
         type: :string,
+        expected: :boolean,
+        reversal: :boolean,
         reference: :string,
         end_to_end_reference: :string,
         settled_at: :string

spec/business_processes/import_bank_statement_spec.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "active_support/all"
-require "cmxl"
 
 require_relative "../../box/models/account"
 require_relative "../../box/business_processes/import_bank_statement"

spec/business_processes/import_statement_spec.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "active_support/all"
-require "cmxl"
 
 require_relative "../../box/models/account"
 require_relative "../../box/models/organization"
@@ -257,6 +256,19 @@ def exec_link_action
             expect(described_class.create_statement(mt940_bank_statement, transaction)).to be_truthy
           end
         end
+
+        context "importing vmk transaction that is expected" do
+          it "marks statements as expected" do
+            vmk_data = File.read("spec/fixtures/master_diesel_vmk_data.txt")
+            bank_statement = ImportBankStatement.from_mt940(vmk_data, account)
+            bank_transactions = described_class.parse_bank_statement(bank_statement)
+            transaction = bank_transactions.first
+
+            described_class.create_statement(bank_statement, transaction)
+
+            expect(Statement.first).to be_expected
+          end
+        end
       end
     end
   end

spec/fabricators/statement_fabricator.rb

@@ -44,4 +44,6 @@
   bic { STATEMENT_BICS.sample }
   eref { "eref-#{Fabricate.sequence(:eref)}" }
   svwz { Faker::Lorem.sentence }
+  expected { [true, false].sample }
+  reversal { [true, false].sample }
 end

spec/fixtures/abnamro.mt940

@@ -0,0 +1,41 @@
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19321/1
+:60F:C110522EUR3236,28
+:61:1105240524D9,N192NONREF
+:86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+INCL. 1,44 BTW
+:61:1105210523D11,59N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+:61:1105230523D11,63N426NONREF
+:86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+:61:1105220523D11,8N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+:61:1105210523D13,45N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+:61:1105210523D15,49N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+:61:1105210523D107,N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+:61:1105220523D141,48N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110523EUR876,84
+-
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19322/1
+:60F:C110523EUR2876,84
+:61:1105240524D9,49N426NONREF
+:86:BEA   NR:XXX1234   24.05.11/09.18 PETS PLACE KATWIJK KATWI,PAS999
+:61:1105240524D15,N426NONREF
+:86:52.89.39.882 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110524EUR1849,75
+-

spec/fixtures/handelbank.mt940

@@ -0,0 +1,16 @@
+ä4:
+:20:5566778899100112
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å
+ä4:
+:20:5566778899100169
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å