perf: make banking syncs incremental on subsequent runs (#141)

## Summary - Subsequent syncs (every 6h) now only process recent data instead of re-syncing full history, reducing unnecessary API calls and database writes - Full sync still runs automatically on first connection and can be forced anytime with `banking:sync --full` - Centralizes `isFirstSync` logic in `SyncBankingConnectionJob` and propagates the `fullSync` flag through the entire chain: Command → `SyncAllBankingConnectionsJob` → `SyncBankingConnectionJob` → provider services ## Changes by provider - **Indexa Capital**: Skips portfolio entries older than the last recorded balance date on incremental syncs (the API doesn't support date filtering, so filtering is done client-side) - **Binance**: Reuses stored `invested_amount` from the database on subsequent syncs instead of fetching up to 2 years of deposit/withdrawal history in 90-day windows - **EnableBanking / Bitpanda**: Already minimal — no changes needed ## Testing - Fixed 6 existing Binance tests to pass `isFirstSync: true` for invested amount calculation - Added 7 new tests covering incremental sync behavior, full sync override, and `--full` flag propagation
2026-02-23 15:10:40 +01:00 · 2026-02-23 15:10:40 +01:00 · d48fea15b2
parent 299b8a56d8
commit d48fea15b2
9 changed files with 264 additions and 26 deletions
--- a/app/Console/Commands/SyncBankingConnections.php
+++ b/app/Console/Commands/SyncBankingConnections.php
@ -14,7 +14,8 @@ class SyncBankingConnections extends Command
    protected $signature = 'banking:sync
        {--user= : Filter by user email address}
        {--connection= : Filter by banking connection ID}
-        {--sync : Run synchronously instead of dispatching to the queue}';
+        {--sync : Run synchronously instead of dispatching to the queue}
+        {--full : Force a full sync instead of incremental}';

    protected $description = 'Sync transactions and balances for all active banking connections';

@ -23,6 +24,7 @@ class SyncBankingConnections extends Command
        $userEmail = $this->option('user');
        $connectionId = $this->option('connection');
        $sync = $this->option('sync');
+        $fullSync = $this->option('full');

        if (! $userEmail && ! $connectionId) {
            if ($sync) {
@ -31,7 +33,7 @@ class SyncBankingConnections extends Command
                return Command::FAILURE;
            }

-            SyncAllBankingConnectionsJob::dispatch();
+            SyncAllBankingConnectionsJob::dispatch($fullSync);

            $this->info('Banking sync jobs dispatched for all active connections.');

@ -69,13 +71,13 @@ class SyncBankingConnections extends Command
            return Command::SUCCESS;
        }

-        $connections->each(function (BankingConnection $connection) use ($sync) {
+        $connections->each(function (BankingConnection $connection) use ($sync, $fullSync) {
            if ($sync) {
                $this->info("Syncing {$connection->provider} connection {$connection->id}...");
-                SyncBankingConnectionJob::dispatchSync($connection);
+                SyncBankingConnectionJob::dispatchSync($connection, $fullSync);
                $this->info("Finished syncing {$connection->provider} connection {$connection->id}.");
            } else {
-                SyncBankingConnectionJob::dispatch($connection);
+                SyncBankingConnectionJob::dispatch($connection, $fullSync);
            }
        });

--- a/app/Jobs/SyncAllBankingConnectionsJob.php
+++ b/app/Jobs/SyncAllBankingConnectionsJob.php
@ -14,6 +14,10 @@ class SyncAllBankingConnectionsJob implements ShouldQueue
 {
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

+    public function __construct(
+        public bool $fullSync = false,
+    ) {}
+
    public function handle(): void
    {
        BankingConnection::query()
@ -23,7 +27,7 @@ class SyncAllBankingConnectionsJob implements ShouldQueue
                    ->orWhere('valid_until', '>', now());
            })
            ->each(function (BankingConnection $connection) {
-                SyncBankingConnectionJob::dispatch($connection);
+                SyncBankingConnectionJob::dispatch($connection, $this->fullSync);
            });
    }
 }
--- a/app/Jobs/SyncBankingConnectionJob.php
+++ b/app/Jobs/SyncBankingConnectionJob.php
@ -33,6 +33,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue

    public function __construct(
        public BankingConnection $bankingConnection,
+        public bool $fullSync = false,
    ) {}

    public function uniqueId(): string
@ -56,14 +57,16 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
        }

        try {
+            $isFirstSync = ! $connection->last_synced_at || $this->fullSync;
+
            if ($connection->isIndexaCapital()) {
-                $this->syncIndexaCapital($connection);
+                $this->syncIndexaCapital($connection, $isFirstSync);
            } elseif ($connection->isBinance()) {
-                $this->syncBinance($connection);
+                $this->syncBinance($connection, $isFirstSync);
            } elseif ($connection->isBitpanda()) {
                $this->syncBitpanda($connection);
            } else {
-                $this->syncEnableBanking($connection, $transactionSync, $balanceSync);
+                $this->syncEnableBanking($connection, $transactionSync, $balanceSync, $isFirstSync);
            }

            $connection->update([
@ -85,7 +88,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
        }
    }

-    private function syncIndexaCapital(BankingConnection $connection): void
+    private function syncIndexaCapital(BankingConnection $connection, bool $isFirstSync): void
    {
        $client = new IndexaCapitalClient($connection->api_token);
        $syncService = new IndexaCapitalBalanceSyncService;
@ -93,13 +96,12 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
        $connection->load('accounts');

        foreach ($connection->accounts as $account) {
-            $syncService->sync($account, $client);
+            $syncService->sync($account, $client, $isFirstSync);
        }
    }

-    private function syncBinance(BankingConnection $connection): void
+    private function syncBinance(BankingConnection $connection, bool $isFirstSync): void
    {
-        $isFirstSync = ! $connection->last_synced_at;
        $client = new BinanceClient($connection->api_token, $connection->api_secret);
        $syncService = app(BinanceBalanceSyncService::class);

@ -127,9 +129,8 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
        }
    }

-    private function syncEnableBanking(BankingConnection $connection, TransactionSyncService $transactionSync, BalanceSyncService $balanceSync): void
+    private function syncEnableBanking(BankingConnection $connection, TransactionSyncService $transactionSync, BalanceSyncService $balanceSync, bool $isFirstSync): void
    {
-        $isFirstSync = ! $connection->last_synced_at;
        $dateFrom = $isFirstSync
            ? now()->subYear()->toDateString()
            : $connection->last_synced_at->toDateString();
--- a/app/Services/Banking/BinanceBalanceSyncService.php
+++ b/app/Services/Banking/BinanceBalanceSyncService.php
@ -53,9 +53,14 @@ class BinanceBalanceSyncService
            Sleep::for(self::THROTTLE_SECONDS)->seconds();
        }

-        $investedAmountCents = $this->calculateInvestedAmount($account, $client);
+        $investedAmountCents = null;

-        Sleep::for(self::THROTTLE_SECONDS)->seconds();
+        if ($isFirstSync) {
+            $investedAmountCents = $this->calculateInvestedAmount($account, $client);
+            Sleep::for(self::THROTTLE_SECONDS)->seconds();
+        } else {
+            $investedAmountCents = $this->getLastInvestedAmount($account);
+        }

        $this->syncCurrentBalance($account, $client, $investedAmountCents);
    }
@ -293,6 +298,17 @@ class BinanceBalanceSyncService
        return 0.0;
    }

+    /**
+     * Get the last stored invested amount for the account.
+     */
+    private function getLastInvestedAmount(Account $account): ?int
+    {
+        return $account->balances()
+            ->whereNotNull('invested_amount')
+            ->latest('balance_date')
+            ->value('invested_amount');
+    }
+
    /**
     * Calculate the net invested amount by fetching all deposit and withdrawal history.
     * Net invested = sum of completed deposits - sum of completed withdrawals, converted to fiat.
--- a/app/Services/Banking/IndexaCapitalBalanceSyncService.php
+++ b/app/Services/Banking/IndexaCapitalBalanceSyncService.php
@ -9,9 +9,10 @@ class IndexaCapitalBalanceSyncService
 {
    /**
     * Sync portfolio balances for an Indexa Capital account.
-     * Stores up to one year of daily historical balances from the portfolios data.
+     * On first sync, stores all available daily historical balances.
+     * On subsequent syncs, only processes entries since the last recorded balance.
     */
-    public function sync(Account $account, IndexaCapitalClient $client): void
+    public function sync(Account $account, IndexaCapitalClient $client, bool $isFirstSync = true): void
    {
        if (! $account->external_account_id) {
            return;
@ -29,6 +30,16 @@ class IndexaCapitalBalanceSyncService
            return;
        }

+        $sinceDate = null;
+
+        if (! $isFirstSync) {
+            $lastBalanceDate = $account->balances()->max('balance_date');
+
+            if ($lastBalanceDate) {
+                $sinceDate = $lastBalanceDate;
+            }
+        }
+
        $count = 0;

        foreach ($portfolios as $entry) {
@ -39,6 +50,10 @@ class IndexaCapitalBalanceSyncService
                continue;
            }

+            if ($sinceDate !== null && $date < $sinceDate) {
+                continue;
+            }
+
            $balanceCents = (int) round(floatval($value) * 100);
            $investedAmountCents = $this->calculateInvestedAmount($entry);

@ -56,6 +71,7 @@ class IndexaCapitalBalanceSyncService
        Log::info('Synced Indexa Capital balances', [
            'account_id' => $account->id,
            'days_synced' => $count,
+            ...($sinceDate ? ['since_date' => $sinceDate] : []),
        ]);
    }

--- a/tests/Feature/OpenBanking/BinanceBalanceSyncTest.php
+++ b/tests/Feature/OpenBanking/BinanceBalanceSyncTest.php
@ -502,7 +502,7 @@ test('calculates invested_amount from deposit and withdrawal history', function

    $client = new BinanceClient('test-key', 'test-secret');
    $service = app(BinanceBalanceSyncService::class);
-    $service->sync($account, $client);
+    $service->sync($account, $client, isFirstSync: true);

    $balance = $account->balances()->first();
    // Deposits: 0.5 BTC → converted via CurrencyConversionService (0.5 / 0.00002 = 25000 EUR)
@ -560,7 +560,7 @@ test('excludes internal transfers from invested_amount calculation', function ()

    $client = new BinanceClient('test-key', 'test-secret');
    $service = app(BinanceBalanceSyncService::class);
-    $service->sync($account, $client);
+    $service->sync($account, $client, isFirstSync: true);

    $balance = $account->balances()->first();
    // Only the external deposit of 500 EUR should count, internal 1000 EUR is excluded
@ -631,7 +631,7 @@ test('filters deposits by status 1 and withdrawals by status 6', function () {

    $client = new BinanceClient('test-key', 'test-secret');
    $service = app(BinanceBalanceSyncService::class);
-    $service->sync($account, $client);
+    $service->sync($account, $client, isFirstSync: true);

    $balance = $account->balances()->first();
    // Deposits: 1000 EUR (completed) — pending 2000 excluded
@ -730,7 +730,7 @@ test('converts stablecoin deposits to fiat for invested_amount', function () {

    $client = new BinanceClient('test-key', 'test-secret');
    $service = app(BinanceBalanceSyncService::class);
-    $service->sync($account, $client);
+    $service->sync($account, $client, isFirstSync: true);

    $balance = $account->balances()->first();
    // Stablecoins USDT and USDC are treated as USD
@ -790,7 +790,7 @@ test('paginates within a window when deposit history hits the 1000-record limit'

    $client = new BinanceClient('test-key', 'test-secret');
    $service = app(BinanceBalanceSyncService::class);
-    $service->sync($account, $client);
+    $service->sync($account, $client, isFirstSync: true);

    $balance = $account->balances()->first();
    // 1000 deposits of 1 EUR + 1 deposit of 500 EUR = 1500 EUR → 150000 cents
@ -836,9 +836,51 @@ test('fetches deposits from older windows when recent window is empty', function

    $client = new BinanceClient('test-key', 'test-secret');
    $service = app(BinanceBalanceSyncService::class);
-    $service->sync($account, $client);
+    $service->sync($account, $client, isFirstSync: true);

    $balance = $account->balances()->first();
    // The deposit from the older window should be found: 1000 EUR → 100000 cents
    expect($balance->invested_amount)->toBe(100000);
 });
+
+test('subsequent sync reuses last invested_amount instead of recalculating', function () {
+    $user = User::factory()->onboarded()->create(['currency_code' => 'EUR']);
+    $connection = BankingConnection::factory()->binance()->create([
+        'user_id' => $user->id,
+    ]);
+    $account = Account::factory()->connected()->create([
+        'user_id' => $user->id,
+        'banking_connection_id' => $connection->id,
+        'external_account_id' => 'binance-portfolio',
+        'currency_code' => 'EUR',
+    ]);
+
+    // Pre-existing balance with invested_amount from a previous first sync
+    $account->balances()->create([
+        'balance_date' => now()->subDay()->toDateString(),
+        'balance' => 5000000,
+        'invested_amount' => 300000, // 3000 EUR
+    ]);
+
+    Http::fake([
+        // No deposit/withdrawal API calls should be made on subsequent sync
+        'api.binance.com/sapi/v1/accountSnapshot*' => Http::response(['snapshotVos' => []]),
+        'api.binance.com/api/v3/account*' => Http::response([
+            'balances' => [
+                ['asset' => 'BTC', 'free' => '1.0', 'locked' => '0.0'],
+            ],
+        ]),
+        'api.binance.com/api/v3/ticker/price' => Http::response([
+            ['symbol' => 'BTCEUR', 'price' => '50000.00'],
+        ]),
+    ]);
+
+    $client = new BinanceClient('test-key', 'test-secret');
+    $service = app(BinanceBalanceSyncService::class);
+    $service->sync($account, $client, isFirstSync: false);
+
+    $todayBalance = $account->balances()->where('balance_date', now()->toDateString())->first();
+    expect($todayBalance->balance)->toBe(5000000);
+    // Should carry forward the last invested_amount
+    expect($todayBalance->invested_amount)->toBe(300000);
+});
--- a/tests/Feature/OpenBanking/IndexaCapitalBalanceSyncTest.php
+++ b/tests/Feature/OpenBanking/IndexaCapitalBalanceSyncTest.php
@ -271,3 +271,99 @@ test('falls back to total_amount minus return when instruments_cost is missing',
    // invested_amount = total_amount - return = 8000 - (-500) = 8500 → 850000 cents
    expect($balance->invested_amount)->toBe(850000);
 });
+
+test('subsequent sync only processes entries since last balance date', function () {
+    $user = User::factory()->onboarded()->create();
+    $connection = BankingConnection::factory()->indexaCapital()->create([
+        'user_id' => $user->id,
+    ]);
+    $account = Account::factory()->connected()->create([
+        'user_id' => $user->id,
+        'banking_connection_id' => $connection->id,
+        'external_account_id' => 'IC-001',
+    ]);
+
+    // Simulate existing balances from a previous full sync
+    $account->balances()->create([
+        'balance_date' => now()->subDays(5)->toDateString(),
+        'balance' => 1400000,
+    ]);
+    $account->balances()->create([
+        'balance_date' => now()->subDays(4)->toDateString(),
+        'balance' => 1410000,
+    ]);
+
+    Http::fake([
+        'api.indexacapital.com/accounts/IC-001/performance' => Http::response([
+            'portfolios' => [
+                // Old entries that should be skipped
+                ['date' => now()->subDays(10)->toDateString(), 'total_amount' => 13000.00],
+                ['date' => now()->subDays(6)->toDateString(), 'total_amount' => 13900.00],
+                // Entry on the last balance date (should be processed — updated)
+                ['date' => now()->subDays(4)->toDateString(), 'total_amount' => 14200.00],
+                // New entries
+                ['date' => now()->subDays(3)->toDateString(), 'total_amount' => 14500.00],
+                ['date' => now()->subDays(2)->toDateString(), 'total_amount' => 14800.00],
+                ['date' => now()->toDateString(), 'total_amount' => 15000.00],
+            ],
+        ]),
+    ]);
+
+    $client = new IndexaCapitalClient('test-token');
+    $service = new IndexaCapitalBalanceSyncService;
+    $service->sync($account, $client, isFirstSync: false);
+
+    // 2 pre-existing + 3 new entries = 5 total (the one on the boundary date gets updated, not duplicated)
+    expect($account->balances()->count())->toBe(5);
+
+    // Verify old entry was NOT overwritten (the one from subDays(5) should still be there)
+    $oldBalance = $account->balances()->where('balance_date', now()->subDays(5)->toDateString())->first();
+    expect($oldBalance->balance)->toBe(1400000);
+
+    // Verify boundary entry was updated
+    $boundaryBalance = $account->balances()->where('balance_date', now()->subDays(4)->toDateString())->first();
+    expect($boundaryBalance->balance)->toBe(1420000);
+
+    // Verify new entry was created
+    $newBalance = $account->balances()->where('balance_date', now()->toDateString())->first();
+    expect($newBalance->balance)->toBe(1500000);
+});
+
+test('full sync processes all entries regardless of existing balances', function () {
+    $user = User::factory()->onboarded()->create();
+    $connection = BankingConnection::factory()->indexaCapital()->create([
+        'user_id' => $user->id,
+    ]);
+    $account = Account::factory()->connected()->create([
+        'user_id' => $user->id,
+        'banking_connection_id' => $connection->id,
+        'external_account_id' => 'IC-001',
+    ]);
+
+    // Simulate existing balances from a previous sync
+    $account->balances()->create([
+        'balance_date' => now()->subDays(2)->toDateString(),
+        'balance' => 1400000,
+    ]);
+
+    Http::fake([
+        'api.indexacapital.com/accounts/IC-001/performance' => Http::response([
+            'portfolios' => [
+                ['date' => now()->subDays(5)->toDateString(), 'total_amount' => 13000.00],
+                ['date' => now()->subDays(2)->toDateString(), 'total_amount' => 14000.00],
+                ['date' => now()->toDateString(), 'total_amount' => 15000.00],
+            ],
+        ]),
+    ]);
+
+    $client = new IndexaCapitalClient('test-token');
+    $service = new IndexaCapitalBalanceSyncService;
+    $service->sync($account, $client, isFirstSync: true);
+
+    // All 3 entries processed (1 existing updated + 2 new)
+    expect($account->balances()->count())->toBe(3);
+
+    // The old entry at subDays(2) should be updated with new value
+    $updatedBalance = $account->balances()->where('balance_date', now()->subDays(2)->toDateString())->first();
+    expect($updatedBalance->balance)->toBe(1400000);
+});
--- a/tests/Feature/OpenBanking/SyncBankingConnectionJobTest.php
+++ b/tests/Feature/OpenBanking/SyncBankingConnectionJobTest.php
@ -433,6 +433,7 @@ test('binance first sync gets current balance immediately and dispatches histori
 });

 test('binance subsequent sync does not dispatch historical job', function () {
+    Mail::fake();
    Queue::fake(SyncBinanceHistoricalBalancesJob::class);

    $user = User::factory()->onboarded()->create(['currency_code' => 'EUR']);
@ -467,7 +468,30 @@ test('binance subsequent sync does not dispatch historical job', function () {
    $job = new SyncBankingConnectionJob($connection);
    $job->handle($transactionSync, $balanceSync);

-    Queue::assertNotPushed(SyncBinanceHistoricalBalancesJob::class);
+    Mail::assertNothingQueued();
+});
+
+test('fullSync flag forces first-sync behavior on already-synced connection', function () {
+    $user = User::factory()->onboarded()->create();
+    $connection = BankingConnection::factory()->create([
+        'user_id' => $user->id,
+        'last_synced_at' => now()->subDay(),
+    ]);
+    $account = Account::factory()->connected()->create([
+        'user_id' => $user->id,
+        'banking_connection_id' => $connection->id,
+        'external_account_id' => 'ext-123',
+    ]);
+
+    $transactionSync = Mockery::mock(TransactionSyncService::class);
+    $transactionSync->shouldReceive('sync')->once()->andReturn(0);
+
+    $balanceSync = Mockery::mock(BalanceSyncService::class);
+    $balanceSync->shouldReceive('sync')->once();
+    $balanceSync->shouldReceive('calculateHistoricalBalances')->once();
+
+    $job = new SyncBankingConnectionJob($connection, fullSync: true);
+    $job->handle($transactionSync, $balanceSync);
 });

 test('bitpanda sync calls balance sync service and updates last_synced_at', function () {
--- a/tests/Feature/SyncBankingConnectionsCommandTest.php
+++ b/tests/Feature/SyncBankingConnectionsCommandTest.php
@ -103,3 +103,40 @@ test('banking:sync can combine user and connection filters', function () {
        return $job->bankingConnection->id === $connection->id;
    });
 });
+
+test('banking:sync passes fullSync flag when --full is provided', function () {
+    Queue::fake();
+
+    artisan('banking:sync', ['--full' => true])
+        ->expectsOutputToContain('Banking sync jobs dispatched for all active connections.')
+        ->assertSuccessful();
+
+    Queue::assertPushed(SyncAllBankingConnectionsJob::class, function ($job) {
+        return $job->fullSync === true;
+    });
+});
+
+test('banking:sync passes fullSync to individual connections with --full', function () {
+    Queue::fake();
+
+    $user = User::factory()->create(['email' => 'test@example.com']);
+    BankingConnection::factory()->for($user)->create();
+
+    artisan('banking:sync', ['--user' => 'test@example.com', '--full' => true])
+        ->assertSuccessful();
+
+    Queue::assertPushed(SyncBankingConnectionJob::class, function ($job) {
+        return $job->fullSync === true;
+    });
+});
+
+test('banking:sync does not set fullSync by default', function () {
+    Queue::fake();
+
+    artisan('banking:sync')
+        ->assertSuccessful();
+
+    Queue::assertPushed(SyncAllBankingConnectionsJob::class, function ($job) {
+        return $job->fullSync === false;
+    });
+});