Backfill Coinbase monthly history (#395)

## Summary
- backfill Coinbase portfolio balances for 12 previous months plus today
- retry missing historical backfill on later syncs
- add Coinbase candle pricing with USD fallback and mark Coinbase
accounts as investments

## Tests
- php artisan test --compact
tests/Feature/OpenBanking/CoinbaseBalanceSyncTest.php
tests/Feature/OpenBanking/AccountMappingTest.php
tests/Feature/OpenBanking/CoinbaseControllerTest.php
This commit is contained in:
Víctor Falcón 2026-05-14 10:52:46 +01:00 committed by GitHub
parent b8e04478df
commit 81c0fd4a81
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 540 additions and 19 deletions

View File

@ -76,7 +76,7 @@ class AccountMappingController extends Controller
$pendingAccounts = collect($connection->pending_accounts_data)
->keyBy('uid');
$accountType = ($connection->isIndexaCapital() || $connection->isBinance() || $connection->isBitpanda())
$accountType = ($connection->isIndexaCapital() || $connection->isBinance() || $connection->isBitpanda() || $connection->isCoinbase())
? AccountType::Investment
: AccountType::Checking;

View File

@ -14,7 +14,7 @@ trait CreatesAccountsFromPending
* Auto-create all pending accounts from a banking connection without user interaction.
*
* This resolves the correct account type based on the provider (Investment for
* Indexa Capital, Binance, and Bitpanda; Checking for everything else) and clears the
* Indexa Capital, Binance, Bitpanda, and Coinbase; Checking for everything else) and clears the
* pending data once accounts have been created.
*/
private function createAccountsFromPending(User $user, BankingConnection $connection): void
@ -28,7 +28,7 @@ trait CreatesAccountsFromPending
$bank->update(['logo' => $connection->aspsp_logo]);
}
$accountType = ($connection->isIndexaCapital() || $connection->isBinance() || $connection->isBitpanda())
$accountType = ($connection->isIndexaCapital() || $connection->isBinance() || $connection->isBitpanda() || $connection->isCoinbase())
? AccountType::Investment
: AccountType::Checking;

View File

@ -115,7 +115,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
} elseif ($connection->isBitpanda()) {
$this->syncBitpanda($connection);
} elseif ($connection->isCoinbase()) {
$this->syncCoinbase($connection);
$this->syncCoinbase($connection, $isFirstSync);
} else {
$metadata = $this->syncEnableBanking($connection, $transactionSync, $balanceSync, $isFirstSync);
@ -331,7 +331,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
}
}
private function syncCoinbase(BankingConnection $connection): void
private function syncCoinbase(BankingConnection $connection, bool $isFirstSync): void
{
$client = new CoinbaseClient($connection->api_token, $connection->api_secret);
$syncService = app(CoinbaseBalanceSyncService::class);
@ -339,7 +339,7 @@ class SyncBankingConnectionJob implements ShouldBeUnique, ShouldQueue
$connection->load('accounts');
foreach ($connection->accounts as $account) {
$syncService->sync($account, $client);
$syncService->sync($account, $client, $isFirstSync, backfillMissingHistory: true);
}
}

View File

@ -4,6 +4,7 @@ namespace App\Services\Banking;
use App\Models\Account;
use App\Services\CurrencyConversionService;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Log;
class CoinbaseBalanceSyncService
@ -13,6 +14,10 @@ class CoinbaseBalanceSyncService
private const USD_CURRENCY = 'USD';
private const HISTORICAL_MONTHS = 12;
private const CANDLE_WINDOW_DAYS = 300;
public function __construct(private CurrencyConversionService $currencyConverter) {}
/**
@ -21,22 +26,34 @@ class CoinbaseBalanceSyncService
*
* @api
*/
public function sync(Account $account, CoinbaseClient $client): void
public function sync(Account $account, CoinbaseClient $client, bool $isFirstSync = false, bool $backfillMissingHistory = false): void
{
if (! $account->external_account_id) {
return;
}
$this->syncCurrentBalance($account, $client);
$coinbaseAccounts = $client->getAllAccounts();
if (empty($coinbaseAccounts)) {
return;
}
if ($isFirstSync || ($backfillMissingHistory && $this->needsHistoricalBackfill($account))) {
$this->syncHistoricalBalances($account, $client, $coinbaseAccounts);
}
$this->syncCurrentBalance($account, $client, $coinbaseAccounts);
}
/**
* Sync today's balance by listing every Coinbase account and converting to target currency.
*
* @param array<int, array<string, mixed>>|null $coinbaseAccounts
*/
public function syncCurrentBalance(Account $account, CoinbaseClient $client): void
public function syncCurrentBalance(Account $account, CoinbaseClient $client, ?array $coinbaseAccounts = null): void
{
$targetCurrency = strtoupper($account->currency_code);
$coinbaseAccounts = $client->getAllAccounts();
$coinbaseAccounts ??= $client->getAllAccounts();
if (empty($coinbaseAccounts)) {
return;
@ -56,6 +73,73 @@ class CoinbaseBalanceSyncService
);
}
/**
* Backfill one year of monthly balances using current holdings valued at each month's matching day close.
*
* @param array<int, array<string, mixed>> $coinbaseAccounts
*/
public function syncHistoricalBalances(Account $account, CoinbaseClient $client, array $coinbaseAccounts): void
{
$targetCurrency = strtoupper($account->currency_code);
$historicalDates = $this->historicalDates();
if ($historicalDates === []) {
return;
}
$startDate = $historicalDates[0];
$endDate = $historicalDates[array_key_last($historicalDates)];
[$fiatBalances, $cryptoAssets] = $this->partitionBalancesByCurrency($coinbaseAccounts);
$priceHistory = $this->fetchHistoricalPriceMaps($client, array_keys($cryptoAssets), $targetCurrency, $startDate, $endDate);
$count = 0;
foreach ($historicalDates as $date) {
$dateString = $date->toDateString();
$totalValue = $this->convertHistoricalFiatBalances($fiatBalances, $targetCurrency, $dateString);
$totalValue += $this->convertHistoricalCryptoAssets($cryptoAssets, $priceHistory, $targetCurrency, $dateString);
if ($totalValue <= 0) {
continue;
}
$account->balances()->updateOrCreate(
['balance_date' => $dateString],
['balance' => (int) round($totalValue * 100)],
);
$count++;
}
Log::info('Synced Coinbase historical balances', [
'account_id' => $account->id,
'days_synced' => $count,
'currency' => $targetCurrency,
]);
}
private function needsHistoricalBackfill(Account $account): bool
{
return ! $account->balances()
->where('balance_date', '<=', $this->historicalStartDate()->toDateString())
->exists();
}
private function historicalStartDate(): Carbon
{
return now()->subMonthsNoOverflow(self::HISTORICAL_MONTHS)->startOfDay();
}
/**
* @return array<int, Carbon>
*/
private function historicalDates(): array
{
return collect(range(self::HISTORICAL_MONTHS, 1))
->map(fn (int $monthsAgo): Carbon => now()->subMonthsNoOverflow($monthsAgo)->startOfDay())
->all();
}
/**
* Split Coinbase accounts into fiat (converted directly) and crypto holdings.
*
@ -64,7 +148,24 @@ class CoinbaseBalanceSyncService
*/
private function partitionBalances(array $coinbaseAccounts, string $targetCurrency): array
{
[$fiatBalances, $cryptoAssets] = $this->partitionBalancesByCurrency($coinbaseAccounts);
$fiatTotal = 0.0;
foreach ($fiatBalances as $currency => $balance) {
$fiatTotal += $this->convertFiat($currency, $balance, $targetCurrency);
}
return [$fiatTotal, $cryptoAssets];
}
/**
* @param array<int, array<string, mixed>> $coinbaseAccounts
* @return array{0: array<string, float>, 1: array<string, float>}
*/
private function partitionBalancesByCurrency(array $coinbaseAccounts): array
{
$fiatBalances = [];
$cryptoAssets = [];
foreach ($coinbaseAccounts as $coinbaseAccount) {
@ -78,7 +179,7 @@ class CoinbaseBalanceSyncService
}
if ($this->isFiatCurrency($currency)) {
$fiatTotal += $this->convertFiat($currency, $balance, $targetCurrency);
$fiatBalances[$currency] = ($fiatBalances[$currency] ?? 0.0) + $balance;
continue;
}
@ -86,7 +187,7 @@ class CoinbaseBalanceSyncService
$cryptoAssets[$currency] = ($cryptoAssets[$currency] ?? 0.0) + $balance;
}
return [$fiatTotal, $cryptoAssets];
return [$fiatBalances, $cryptoAssets];
}
private function convertFiat(string $currency, float $amount, string $targetCurrency): float
@ -152,6 +253,163 @@ class CoinbaseBalanceSyncService
return $map;
}
/**
* Fetch daily close prices keyed by asset and date.
*
* @param array<int, string> $assets
* @return array<string, array<string, float>>
*/
private function fetchHistoricalPriceMaps(CoinbaseClient $client, array $assets, string $targetCurrency, Carbon $startDate, Carbon $endDate): array
{
$priceHistory = [];
foreach ($assets as $asset) {
if (in_array($asset, self::USD_STABLECOINS, true)) {
continue;
}
$priceHistory[$asset] = $this->fetchHistoricalPricesForAsset($client, $asset, $targetCurrency, $startDate, $endDate);
}
return $priceHistory;
}
/**
* @return array<string, float>
*/
private function fetchHistoricalPricesForAsset(CoinbaseClient $client, string $asset, string $targetCurrency, Carbon $startDate, Carbon $endDate): array
{
$directPrices = $this->fetchHistoricalProductPrices($client, "{$asset}-{$targetCurrency}", $startDate, $endDate);
if ($directPrices !== [] || $targetCurrency === self::USD_CURRENCY) {
return $directPrices;
}
$usdPrices = $this->fetchHistoricalProductPrices($client, "{$asset}-".self::USD_CURRENCY, $startDate, $endDate);
$targetPrices = [];
foreach ($usdPrices as $date => $usdPrice) {
$targetPrice = $this->currencyConverter->convert(self::USD_CURRENCY, $targetCurrency, $usdPrice, $date);
if ($targetPrice > 0) {
$targetPrices[$date] = $targetPrice;
}
}
return $targetPrices;
}
/**
* @return array<string, float>
*/
private function fetchHistoricalProductPrices(CoinbaseClient $client, string $productId, Carbon $startDate, Carbon $endDate): array
{
$prices = [];
foreach ($this->fetchProductCandles($client, $productId, $startDate, $endDate) as $candle) {
$timestamp = $candle['start'] ?? null;
$close = (float) ($candle['close'] ?? 0);
if ($timestamp === null || $close <= 0) {
continue;
}
$prices[Carbon::createFromTimestamp((int) $timestamp)->toDateString()] = $close;
}
return $prices;
}
/**
* @return array<int, array<string, mixed>>
*/
private function fetchProductCandles(CoinbaseClient $client, string $productId, Carbon $startDate, Carbon $endDate): array
{
$candles = [];
$windowStart = $startDate->copy();
while ($windowStart->lessThanOrEqualTo($endDate)) {
$windowEnd = $windowStart->copy()->addDays(self::CANDLE_WINDOW_DAYS)->min($endDate);
try {
$response = $client->getProductCandles(
$productId,
$windowStart->getTimestamp(),
$windowEnd->copy()->endOfDay()->getTimestamp(),
);
} catch (\Throwable $e) {
Log::warning('Coinbase historical candles failed', [
'product_id' => $productId,
'start' => $windowStart->toDateString(),
'end' => $windowEnd->toDateString(),
'error' => $e->getMessage(),
]);
break;
}
foreach ($response['candles'] ?? [] as $candle) {
$candles[] = $candle;
}
$windowStart = $windowEnd->copy()->addDay()->startOfDay();
}
return $candles;
}
/**
* @param array<string, float> $fiatBalances
*/
private function convertHistoricalFiatBalances(array $fiatBalances, string $targetCurrency, string $date): float
{
$total = 0.0;
foreach ($fiatBalances as $currency => $amount) {
$total += $this->convertFiatOnDate($currency, $amount, $targetCurrency, $date);
}
return $total;
}
private function convertFiatOnDate(string $currency, float $amount, string $targetCurrency, string $date): float
{
if ($currency === $targetCurrency) {
return $amount;
}
return $this->currencyConverter->convert($currency, $targetCurrency, $amount, $date);
}
/**
* @param array<string, float> $cryptoAssets
* @param array<string, array<string, float>> $priceHistory
*/
private function convertHistoricalCryptoAssets(array $cryptoAssets, array $priceHistory, string $targetCurrency, string $date): float
{
$total = 0.0;
foreach ($cryptoAssets as $asset => $quantity) {
if (in_array($asset, self::USD_STABLECOINS, true)) {
$total += $this->convertFiatOnDate(self::USD_CURRENCY, $quantity, $targetCurrency, $date);
continue;
}
$price = $priceHistory[$asset][$date] ?? null;
if ($price !== null) {
$total += $quantity * $price;
continue;
}
$total += $this->currencyConverter->convert($asset, $targetCurrency, $quantity, $date);
}
return $total;
}
/**
* Convert each crypto holding to target fiat. Falls back via USD pair + currency converter.
*

View File

@ -106,6 +106,20 @@ class CoinbaseClient
return $this->signedRequest('GET', '/api/v3/brokerage/best_bid_ask', $params);
}
/**
* Get historical candles for a product.
*
* @return array<string, mixed>
*/
public function getProductCandles(string $productId, int $start, int $end, string $granularity = 'ONE_DAY'): array
{
return $this->signedRequest('GET', "/api/v3/brokerage/products/{$productId}/candles", [
'start' => $start,
'end' => $end,
'granularity' => $granularity,
]);
}
/**
* Execute a signed JWT request with retry on rate limiting.
*

View File

@ -106,17 +106,17 @@ test('store with action create creates new accounts', function () {
Queue::assertPushed(SyncBankingConnectionJob::class);
});
test('store creates investment accounts for bitpanda connections', function () {
test('store creates investment accounts for crypto provider connections', function (string $provider, string $providerName, string $uid) {
Queue::fake();
$user = User::factory()->onboarded()->create();
$connection = BankingConnection::factory()->awaitingMapping()->create([
'user_id' => $user->id,
'provider' => 'bitpanda',
'aspsp_name' => 'Bitpanda',
'provider' => $provider,
'aspsp_name' => $providerName,
'pending_accounts_data' => [
[
'uid' => 'bitpanda-portfolio',
'uid' => $uid,
'currency' => 'EUR',
'name' => 'Crypto Portfolio',
'account_id' => [],
@ -128,7 +128,7 @@ test('store creates investment accounts for bitpanda connections', function () {
->post(route('open-banking.map-accounts.store', $connection), [
'mappings' => [
[
'bank_account_uid' => 'bitpanda-portfolio',
'bank_account_uid' => $uid,
'action' => 'create',
'existing_account_id' => null,
],
@ -140,12 +140,15 @@ test('store creates investment accounts for bitpanda connections', function () {
$this->assertDatabaseHas('accounts', [
'user_id' => $user->id,
'banking_connection_id' => $connection->id,
'external_account_id' => 'bitpanda-portfolio',
'external_account_id' => $uid,
'type' => 'investment',
]);
Queue::assertPushed(SyncBankingConnectionJob::class);
});
})->with([
'bitpanda' => ['bitpanda', 'Bitpanda', 'bitpanda-portfolio'],
'coinbase' => ['coinbase', 'Coinbase', 'coinbase-portfolio'],
]);
test('store with action link links existing account', function () {
Queue::fake();

View File

@ -5,8 +5,12 @@ use App\Models\BankingConnection;
use App\Models\User;
use App\Services\Banking\CoinbaseBalanceSyncService;
use App\Services\Banking\CoinbaseClient;
use Illuminate\Http\Client\Request;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Http;
afterEach(fn () => Carbon::setTestNow());
function ecPrivateKeyForCoinbase(): string
{
$key = openssl_pkey_new([
@ -80,6 +84,248 @@ test('syncs coinbase balance with crypto and fiat wallets', function () {
expect($balance->balance_date->toDateString())->toBe(now()->toDateString());
});
test('first sync creates twelve monthly coinbase historical balances', function () {
Carbon::setTestNow('2026-05-14 12:00:00');
$user = User::factory()->onboarded()->create(['currency_code' => 'EUR']);
$connection = BankingConnection::factory()->coinbase()->create([
'user_id' => $user->id,
]);
$account = Account::factory()->connected()->create([
'user_id' => $user->id,
'banking_connection_id' => $connection->id,
'external_account_id' => 'coinbase-portfolio',
'currency_code' => 'EUR',
]);
Http::fake(function (Request $request) {
$url = $request->url();
if (str_contains($url, '/api/v3/brokerage/accounts')) {
return Http::response([
'accounts' => [
[
'uuid' => 'cb-1',
'name' => 'BTC',
'currency' => 'BTC',
'available_balance' => ['value' => '1.0', 'currency' => 'BTC'],
'hold' => ['value' => '0', 'currency' => 'BTC'],
'active' => true,
'type' => 'ACCOUNT_TYPE_CRYPTO',
],
[
'uuid' => 'cb-2',
'name' => 'EUR',
'currency' => 'EUR',
'available_balance' => ['value' => '50.00', 'currency' => 'EUR'],
'hold' => ['value' => '0', 'currency' => 'EUR'],
'active' => true,
'type' => 'ACCOUNT_TYPE_FIAT',
],
],
'has_next' => false,
'cursor' => '',
'size' => 2,
]);
}
if (str_contains($url, '/api/v3/brokerage/products/BTC-EUR/candles')) {
parse_str(parse_url($url, PHP_URL_QUERY) ?: '', $query);
$start = Carbon::createFromTimestamp((int) $query['start'])->startOfDay();
$end = Carbon::createFromTimestamp((int) $query['end'])->startOfDay();
$candles = [];
for ($date = $start->copy(); $date->lessThanOrEqualTo($end); $date->addDay()) {
$candles[] = [
'start' => (string) $date->getTimestamp(),
'low' => '100.00',
'high' => '100.00',
'open' => '100.00',
'close' => '100.00',
'volume' => '1.00',
];
}
return Http::response(['candles' => $candles]);
}
if (str_contains($url, '/api/v3/brokerage/best_bid_ask')) {
return Http::response([
'pricebooks' => [
[
'product_id' => 'BTC-EUR',
'bids' => [['price' => '200.00', 'size' => '1']],
'asks' => [['price' => '200.00', 'size' => '1']],
],
],
]);
}
return Http::response([], 404);
});
$client = new CoinbaseClient('organizations/org/apiKeys/key', ecPrivateKeyForCoinbase());
$service = app(CoinbaseBalanceSyncService::class);
$service->sync($account, $client, isFirstSync: true);
$balances = $account->balances()->orderBy('balance_date')->get();
expect($balances)->toHaveCount(13);
expect($balances->first()->balance_date->toDateString())->toBe('2025-05-14');
expect($balances->first()->balance)->toBe(15_000);
expect($balances->get(11)->balance_date->toDateString())->toBe('2026-04-14');
expect($balances->get(11)->balance)->toBe(15_000);
expect($balances->last()->balance_date->toDateString())->toBe('2026-05-14');
expect($balances->last()->balance)->toBe(25_000);
});
test('historical sync falls back to usd coinbase candles when target pair is unavailable', function () {
Carbon::setTestNow('2026-05-14 12:00:00');
$user = User::factory()->onboarded()->create(['currency_code' => 'EUR']);
$connection = BankingConnection::factory()->coinbase()->create([
'user_id' => $user->id,
]);
$account = Account::factory()->connected()->create([
'user_id' => $user->id,
'banking_connection_id' => $connection->id,
'external_account_id' => 'coinbase-portfolio',
'currency_code' => 'EUR',
]);
Http::fake(function (Request $request) {
$url = $request->url();
if (str_contains($url, '/api/v3/brokerage/accounts')) {
return Http::response([
'accounts' => [
[
'uuid' => 'cb-1',
'name' => 'SOL',
'currency' => 'SOL',
'available_balance' => ['value' => '2.0', 'currency' => 'SOL'],
'hold' => ['value' => '0', 'currency' => 'SOL'],
'active' => true,
'type' => 'ACCOUNT_TYPE_CRYPTO',
],
],
'has_next' => false,
'cursor' => '',
'size' => 1,
]);
}
if (str_contains($url, '/api/v3/brokerage/products/SOL-EUR/candles')) {
return Http::response(['error' => 'not found'], 404);
}
if (str_contains($url, '/api/v3/brokerage/products/SOL-USD/candles')) {
parse_str(parse_url($url, PHP_URL_QUERY) ?: '', $query);
$start = Carbon::createFromTimestamp((int) $query['start'])->startOfDay();
$end = Carbon::createFromTimestamp((int) $query['end'])->startOfDay();
$candles = [];
for ($date = $start->copy(); $date->lessThanOrEqualTo($end); $date->addDay()) {
$candles[] = [
'start' => (string) $date->getTimestamp(),
'low' => '100.00',
'high' => '100.00',
'open' => '100.00',
'close' => '100.00',
'volume' => '1.00',
];
}
return Http::response(['candles' => $candles]);
}
if (str_contains($url, 'cdn.jsdelivr.net') || str_contains($url, 'currency-api.pages.dev')) {
return Http::response(['eur' => ['usd' => 2.0]]);
}
if (str_contains($url, '/api/v3/brokerage/best_bid_ask')) {
return Http::response([
'pricebooks' => [
[
'product_id' => 'SOL-EUR',
'bids' => [['price' => '50.00', 'size' => '1']],
'asks' => [['price' => '50.00', 'size' => '1']],
],
],
]);
}
return Http::response([], 404);
});
$client = new CoinbaseClient('organizations/org/apiKeys/key', ecPrivateKeyForCoinbase());
$service = app(CoinbaseBalanceSyncService::class);
$service->sync($account, $client, isFirstSync: true);
$balances = $account->balances()->orderBy('balance_date')->get();
expect($balances)->toHaveCount(13);
expect($balances->first()->balance)->toBe(10_000);
expect($balances->last()->balance)->toBe(10_000);
});
test('subsequent sync backfills coinbase monthly history when missing', function () {
Carbon::setTestNow('2026-05-14 12:00:00');
$user = User::factory()->onboarded()->create(['currency_code' => 'USD']);
$connection = BankingConnection::factory()->coinbase()->create([
'user_id' => $user->id,
'last_synced_at' => now()->subHour(),
]);
$account = Account::factory()->connected()->create([
'user_id' => $user->id,
'banking_connection_id' => $connection->id,
'external_account_id' => 'coinbase-portfolio',
'currency_code' => 'USD',
]);
$account->balances()->create([
'balance_date' => now()->toDateString(),
'balance' => 100_000,
]);
Http::fake([
'api.coinbase.com/api/v3/brokerage/accounts*' => Http::response([
'accounts' => [
[
'uuid' => 'cb-1',
'name' => 'USDC',
'currency' => 'USDC',
'available_balance' => ['value' => '1000.00', 'currency' => 'USDC'],
'hold' => ['value' => '0', 'currency' => 'USDC'],
'active' => true,
'type' => 'ACCOUNT_TYPE_CRYPTO',
],
],
'has_next' => false,
'cursor' => '',
'size' => 1,
]),
'api.coinbase.com/api/v3/brokerage/best_bid_ask*' => Http::response([
'pricebooks' => [],
]),
]);
$client = new CoinbaseClient('organizations/org/apiKeys/key', ecPrivateKeyForCoinbase());
$service = app(CoinbaseBalanceSyncService::class);
$service->sync($account, $client, isFirstSync: false, backfillMissingHistory: true);
$balances = $account->balances()->orderBy('balance_date')->get();
expect($balances)->toHaveCount(13);
expect($balances->first()->balance_date->toDateString())->toBe('2025-05-14');
expect($balances->first()->balance)->toBe(100_000);
expect($balances->get(11)->balance_date->toDateString())->toBe('2026-04-14');
expect($balances->last()->balance_date->toDateString())->toBe('2026-05-14');
});
test('treats usd stablecoins as usd when valuing', function () {
$user = User::factory()->onboarded()->create(['currency_code' => 'USD']);
$connection = BankingConnection::factory()->coinbase()->create([