Improve map providers (#996)

* add new providers

* adding id to clusters

* fix lint error
This commit is contained in:
Blake Kaufman
2026-07-06 10:47:40 -04:00
committed by GitHub
parent baa08633dc
commit 8c1c2b5a16
19 changed files with 360 additions and 41 deletions
@@ -0,0 +1,57 @@
// dedupeMerge collapses the same physical merchant appearing across map sources
// (BTC Map + the aux directories) into a single pin, keeping the highest-priority
// record and backfilling fields the winner is missing.
const { dedupeMerge } = require('../../../app/functions/btcMap/mergePlaces');
describe('dedupeMerge', () => {
it('collapses near-identical places from different sources into one row', () => {
const rows = [
{ id: 1, source: 'btcmap', lat: 9.9281, lon: -84.0907, name: "Joe's Coffee", icon: 'local_cafe', category: null },
{ id: 'joes-coffee-4', source: 'bitcoinjungle', lat: 9.92809, lon: -84.09071, name: 'Joes Coffee', icon: '', category: 'food_drink' },
];
const merged = dedupeMerge(rows);
expect(merged).toHaveLength(1);
});
it('keeps the higher-priority source (btcmap) as the winner', () => {
const rows = [
{ id: 'x-1', source: 'moneybadger', lat: 1, lon: 2, name: 'Shop', icon: '', category: 'retail' },
{ id: 42, source: 'btcmap', lat: 1.00001, lon: 2.00001, name: 'Shop', icon: 'storefront', category: null },
];
const merged = dedupeMerge(rows);
expect(merged).toHaveLength(1);
expect(merged[0].source).toBe('btcmap');
expect(merged[0].id).toBe(42);
});
it('backfills fields the winner is missing from the duplicate', () => {
const rows = [
// btcmap wins but has no category (resolved from icon at render); the aux
// duplicate's category should backfill so filtering still works.
{ id: 7, source: 'btcmap', lat: 5, lon: 6, name: 'Cafe', icon: '', category: null },
{ id: 'cafe-1', source: 'bitcoinjungle', lat: 5, lon: 6, name: 'Cafe', icon: '', category: 'food_drink' },
];
const merged = dedupeMerge(rows);
expect(merged).toHaveLength(1);
expect(merged[0].source).toBe('btcmap');
expect(merged[0].category).toBe('food_drink');
});
it('keeps distinct merchants at different coordinates separate', () => {
const rows = [
{ id: 1, source: 'btcmap', lat: 10, lon: 20, name: 'A', icon: '', category: null },
{ id: 2, source: 'moneybadger', lat: 30, lon: 40, name: 'B', icon: '', category: 'retail' },
];
expect(dedupeMerge(rows)).toHaveLength(2);
});
it('drops rows with non-finite coordinates', () => {
const rows = [
{ id: 1, source: 'btcmap', lat: NaN, lon: 20, name: 'A', icon: '', category: null },
{ id: 2, source: 'moneybadger', lat: 30, lon: 40, name: 'B', icon: '', category: 'retail' },
];
const merged = dedupeMerge(rows);
expect(merged).toHaveLength(1);
expect(merged[0].id).toBe(2);
});
});
@@ -717,6 +717,7 @@ export default function CustomHalfModal(props) {
<BTCMapMerchantContent
handleBackPressFunction={handleBackPressFunction}
placeId={props?.route?.params?.placeId}
source={props?.route?.params?.source}
setContentHeight={setContentHeight}
/>
);
+115
View File
@@ -2,6 +2,7 @@ import { openDatabaseAsync } from 'expo-sqlite';
const DB_NAME = 'btcmap.db';
const PLACES_TABLE = 'btcmap_places';
const PROVIDER_TABLE = 'provider_places';
const META_TABLE = 'btcmap_meta';
const BATCH_SIZE = 250;
@@ -28,6 +29,22 @@ async function openDB() {
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS ${PROVIDER_TABLE} (
source TEXT NOT NULL,
native_id TEXT NOT NULL,
lat REAL NOT NULL,
lon REAL NOT NULL,
icon TEXT DEFAULT '',
name TEXT DEFAULT '',
category TEXT DEFAULT '',
address TEXT,
website TEXT,
phone TEXT,
email TEXT,
lightning_address TEXT,
PRIMARY KEY (source, native_id)
);
CREATE INDEX IF NOT EXISTS idx_pp_lat_lon ON ${PROVIDER_TABLE}(lat, lon);
`);
// Migrate existing installs that predate the `name` column.
try {
@@ -79,6 +96,24 @@ export async function setLastSyncTime(ts) {
[String(ts)],
);
}
// Aux providers (Bitcoin Jungle, MoneyBadger) change slowly, so they sync on
// their own weekly cadence — separate from BTC Map's 4-hour last_sync_time.
export async function getProviderLastSyncTime() {
await openDB();
const row = await db.getFirstAsync(
`SELECT value FROM ${META_TABLE} WHERE key = 'provider_last_sync_time'`,
);
return row ? Number(row.value) : null;
}
export async function setProviderLastSyncTime(ts) {
await openDB();
await db.runAsync(
`INSERT OR REPLACE INTO ${META_TABLE} (key, value) VALUES ('provider_last_sync_time', ?)`,
[String(ts)],
);
}
export async function upsertPlaces(places) {
if (!places.length) return;
await openDB();
@@ -167,10 +202,90 @@ export async function getPlacesInBbox(minLat, maxLat, minLon, maxLon) {
);
}
// --- Aux providers (Bitcoin Jungle, MoneyBadger) ---------------------------
// Full-replace a single source: called once (with `clear: true`) for the first
// batch of a provider's NDJSON, then appended to for subsequent batches.
export async function replaceProviderPlaces(source, rows, { clear } = {}) {
await openDB();
await db.withTransactionAsync(async () => {
if (clear) {
await db.runAsync(`DELETE FROM ${PROVIDER_TABLE} WHERE source = ?`, [
source,
]);
}
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
const placeholders = batch.map(() => '(?,?,?,?,?,?,?,?,?,?,?,?)').join(',');
const values = batch.flatMap(p => [
source,
String(p.native_id),
p.lat,
p.lon,
p.icon || '',
p.name || '',
p.category || '',
p.address ?? null,
p.website ?? null,
p.phone ?? null,
p.email ?? null,
p.lightning_address ?? null,
]);
await db.runAsync(
`INSERT OR REPLACE INTO ${PROVIDER_TABLE}
(source, native_id, lat, lon, icon, name, category, address, website, phone, email, lightning_address)
VALUES ${placeholders}`,
values,
);
}
});
}
export async function getProviderPlace(source, nativeId) {
await openDB();
return db.getFirstAsync(
`SELECT * FROM ${PROVIDER_TABLE} WHERE source = ? AND native_id = ?`,
[source, String(nativeId)],
);
}
// Unioned viewport query. Both tables project to the same row shape
// {id, source, lat, lon, icon, name, category} so clustering/merge/list code is
// source-agnostic. `id` is the per-source native id (numeric for BTC Map, the
// stored native_id for aux) and doubles as the detail lookup key. BTC Map rows
// have no stored category (resolved from `icon` at render); aux rows carry a
// pre-resolved bucket in `category` and a blank `icon`.
export async function getAllPlacesInBbox(minLat, maxLat, minLon, maxLon) {
await openDB();
const [btc, aux] = await Promise.all([
db.getAllAsync(
`SELECT id, lat, lon, icon, name FROM ${PLACES_TABLE}
WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`,
[minLat, maxLat, minLon, maxLon],
),
db.getAllAsync(
`SELECT native_id AS id, source, lat, lon, icon, name, category FROM ${PROVIDER_TABLE}
WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`,
[minLat, maxLat, minLon, maxLon],
),
]);
const btcRows = btc.map(p => ({
id: p.id,
source: 'btcmap',
lat: p.lat,
lon: p.lon,
icon: p.icon,
name: p.name,
category: null,
}));
return btcRows.concat(aux);
}
export const deleteBtcMapTable = async () => {
try {
await openDB();
await db.runAsync(`DROP TABLE IF EXISTS ${PLACES_TABLE};`);
await db.runAsync(`DROP TABLE IF EXISTS ${PROVIDER_TABLE};`);
await db.runAsync(`DROP TABLE IF EXISTS ${META_TABLE};`);
console.log(`btc map places and metadata deleted successfully`);
} catch (error) {
+10
View File
@@ -170,3 +170,13 @@ assign('leisure', [
export function getBtcMapCategory(materialIconName) {
return CATEGORY_BY_ICON[materialIconName] ?? 'other';
}
// Category bucket for a merged viewport place, source-aware: BTC Map derives it
// from the Material icon name; aux providers (bitcoinjungle/moneybadger) carry a
// pre-resolved bucket in `category`.
export function resolvePlaceCategory(place) {
if (place?.source && place.source !== 'btcmap') {
return BTC_MAP_CATEGORIES.includes(place.category) ? place.category : 'other';
}
return getBtcMapCategory(place?.icon);
}
+7 -2
View File
@@ -15,7 +15,11 @@ export class ClusterManager {
load(points) {
const features = points.map(p => ({
type: 'Feature',
properties: { pointId: p.id, icon: p.icon },
properties: {
pointId: p.id,
icon: p.icon,
source: p.source || 'btcmap',
},
geometry: { type: 'Point', coordinates: [p.lon, p.lat] },
}));
this._cluster.load(features);
@@ -43,12 +47,13 @@ export class ClusterManager {
};
}
return {
id: `point-${props.pointId}`,
id: `point-${props.source || 'btcmap'}-${props.pointId}`,
type: 'single',
latitude: lat,
longitude: lon,
count: 1,
placeId: props.pointId,
source: props.source || 'btcmap',
};
});
}
+53
View File
@@ -0,0 +1,53 @@
// Collapse places that the same physical merchant appears as across sources
// (BTC Map + the aux directories). Ported from Buho_go's places.js mergePlaces.
//
// Runs in-memory on a viewport result set (small), so an O(n) Map pass is cheap.
// Earlier sources win conflicts; the winner backfills any fields it is missing
// from lower-priority duplicates so a shared pin keeps the richest record.
const SOURCE_PRIORITY = ['btcmap', 'bitcoinjungle', 'moneybadger'];
const SOURCE_RANK = Object.fromEntries(SOURCE_PRIORITY.map((s, i) => [s, i]));
// Round to 4 decimals (~11m) so coordinates that differ slightly between
// sources still collide, combined with a normalized name token.
function dedupeKey(p) {
const lat = Math.round(p.lat * 10000) / 10000;
const lon = Math.round(p.lon * 10000) / 10000;
const name = (p.name || '')
.toLowerCase()
.replace(/[^a-z0-9]/g, '')
.slice(0, 12);
return `${lat},${lon}|${name}`;
}
const BACKFILL_FIELDS = ['name', 'icon', 'category'];
function rankOf(source) {
const r = SOURCE_RANK[source];
return r === undefined ? SOURCE_PRIORITY.length : r;
}
export function dedupeMerge(rows) {
const map = new Map();
for (const p of rows) {
if (!Number.isFinite(p.lat) || !Number.isFinite(p.lon)) continue;
const key = dedupeKey(p);
const existing = map.get(key);
if (!existing) {
map.set(key, { ...p });
continue;
}
if (rankOf(p.source) < rankOf(existing.source)) {
const winner = { ...p };
for (const f of BACKFILL_FIELDS) {
if (!winner[f] && existing[f]) winner[f] = existing[f];
}
map.set(key, winner);
} else {
for (const f of BACKFILL_FIELDS) {
if (!existing[f] && p[f]) existing[f] = p[f];
}
}
}
return [...map.values()];
}
+1 -1
View File
@@ -91,7 +91,7 @@ export async function initializeSparkSession({
const [balance, sparkAddress, freshIdentityPubKey] = await Promise.all([
skipBalanceFetch
? Promise.resolve({ didWork: false })
: getSparkBalance(mnemonic),
: getBalanceWithTimeout(mnemonic, 10000),
getSparkAddress(mnemonic),
cachedIdentityPubKey
? Promise.resolve(cachedIdentityPubKey)
+3 -2
View File
@@ -19,7 +19,7 @@ import ThemeIcon from '../../functions/CustomElements/themeIcon';
import { COLORS, SHADOWS } from '../../constants/theme';
import ThemeImage from '../../functions/CustomElements/themeImage';
import { cameraToBbox } from '../../functions/btcMap/mapClustering';
import { getBtcMapCategory } from '../../functions/btcMap/iconCategory';
import { resolvePlaceCategory } from '../../functions/btcMap/iconCategory';
import {
clearBTCMapClusterCache,
getOrBuildBTCMapClusterManager,
@@ -199,7 +199,7 @@ export default function BTCMapScreen() {
if (filter.categories.length) {
const categorySet = new Set(filter.categories);
points = points.filter(p => categorySet.has(getBtcMapCategory(p.icon)));
points = points.filter(p => categorySet.has(resolvePlaceCategory(p)));
}
if (!points.length) {
@@ -310,6 +310,7 @@ export default function BTCMapScreen() {
navigate.navigate('CustomHalfModal', {
wantedContent: 'btcMapMerchant',
placeId: data.placeId,
source: data.source,
});
}
},
+5 -4
View File
@@ -15,7 +15,7 @@ import { COLORS, INSET_WINDOW_WIDTH, SIZES } from '../../constants/theme';
import { getBtcMapIcon } from '../../functions/btcMap/iconMaping';
import {
CATEGORY_META,
getBtcMapCategory,
resolvePlaceCategory,
} from '../../functions/btcMap/iconCategory';
import {
distanceMeters,
@@ -72,7 +72,7 @@ export default function BTCMapListContent({
const categorySet = categories.length ? new Set(categories) : null;
let rows = places;
if (categorySet) {
rows = rows.filter(p => categorySet.has(getBtcMapCategory(p.icon)));
rows = rows.filter(p => categorySet.has(resolvePlaceCategory(p)));
}
rows = rows.map(p => ({
...p,
@@ -97,7 +97,7 @@ export default function BTCMapListContent({
const hasMoreRows = data?.length < placeCount;
const renderItem = ({ item }) => {
const category = getBtcMapCategory(item.icon);
const category = resolvePlaceCategory(item);
return (
<TouchableOpacity
activeOpacity={0.6}
@@ -106,6 +106,7 @@ export default function BTCMapListContent({
navigate.push('CustomHalfModal', {
wantedContent: 'btcMapMerchant',
placeId: item.id,
source: item.source,
})
}
>
@@ -154,7 +155,7 @@ export default function BTCMapListContent({
<FlatList
data={data}
renderItem={renderItem}
keyExtractor={item => String(item.id)}
keyExtractor={item => `${item.source}-${item.id}`}
showsVerticalScrollIndicator={false}
contentContainerStyle={{ paddingBottom: bottomPadding }}
ListEmptyComponent={
+4 -3
View File
@@ -25,6 +25,7 @@ import { useTranslation } from 'react-i18next';
export default function BTCMapMerchantContent({
placeId,
source,
setContentHeight,
handleBackPressFunction,
}) {
@@ -55,10 +56,10 @@ export default function BTCMapMerchantContent({
useEffect(() => {
if (!placeId) return;
setDetailLoading(true);
getPlaceDetail(placeId, privateKey, publicKey)
getPlaceDetail(placeId, source, privateKey, publicKey)
.then(detail => setPlace(detail))
.finally(() => setDetailLoading(false));
}, [placeId, getPlaceDetail, privateKey, publicKey]);
}, [placeId, source, getPlaceDetail, privateKey, publicKey]);
const handleDirections = useCallback(() => {
if (!lat || !lon) return;
@@ -252,7 +253,7 @@ export default function BTCMapMerchantContent({
</View>
<ThemeText
content={t('screens.btcMap.merchant.dataMessage')}
content={t('screens.btcMap.merchant.dataMessage', { source })}
styles={styles.attribution}
/>
</View>
+96 -21
View File
@@ -9,16 +9,21 @@ import React, {
import { InteractionManager } from 'react-native';
import {
initBTCMapDB,
getPlacesInBbox,
getAllPlacesInBbox,
getProviderPlace,
replaceProviderPlaces,
getLastModified,
setLastModified,
getLastSyncTime,
setLastSyncTime,
getProviderLastSyncTime,
setProviderLastSyncTime,
upsertPlaces,
deletePlaces,
truncateAndInsertPlaces,
needsToResyncMapsData,
} from '../app/functions/btcMap/btcMapStorage';
import { dedupeMerge } from '../app/functions/btcMap/mergePlaces';
import { clearBTCMapClusterCache } from '../app/functions/btcMap/btcMapClusterCache';
import fetchBackend from '../db/handleBackend';
import { useKeysContext } from './keys';
@@ -26,9 +31,37 @@ import * as Location from 'expo-location';
const DEFAULT_LOCATION = { latitude: 51.5074, longitude: -0.1278 };
const FOUR_HOURS_MS = 4 * 60 * 60 * 1000;
const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000;
const PARSE_BATCH = 250;
const BTCMapContext = createContext(null);
// Parse a provider's NDJSON snapshot and write it to SQLite in small batches,
// yielding to the event loop between batches so a large snapshot (MoneyBadger
// ~5.5k) never blocks the JS thread in one synchronous JSON.parse.
async function ingestProviderNDJSON(source, ndjson) {
if (!ndjson) {
await replaceProviderPlaces(source, [], { clear: true });
return;
}
const lines = ndjson.split('\n');
let clear = true;
for (let i = 0; i < lines.length; i += PARSE_BATCH) {
const batch = [];
const end = Math.min(i + PARSE_BATCH, lines.length);
for (let j = i; j < end; j++) {
const line = lines[j];
if (!line) continue;
try {
batch.push(JSON.parse(line));
} catch (_) {}
}
await replaceProviderPlaces(source, batch, { clear });
clear = false;
await new Promise(res => setTimeout(res, 0));
}
}
export function BTCMapProvider({ children }) {
const [isLoading, setIsLoading] = useState(false);
const [syncError, setSyncError] = useState(null);
@@ -61,11 +94,18 @@ export function BTCMapProvider({ children }) {
setIsLoading(true);
setSyncError(null);
try {
// Aux providers change slowly — only pull them once a week. The client
// tells the backend whether to include them so an off-week sync doesn't
// fetch/encrypt/transfer the provider payload at all.
const providerLastSync = await getProviderLastSyncTime();
const includeProviders =
!providerLastSync || Date.now() - providerLastSync > ONE_WEEK_MS;
const currentTs = await getLastModified();
const requestData =
currentTs && !needsToResync
? { action: 'sync', updated_since: currentTs }
: { action: 'sync' };
? { action: 'sync', updated_since: currentTs, includeProviders }
: { action: 'sync', includeProviders };
const result = await fetchBackend(
'getBTCMapData',
@@ -83,6 +123,17 @@ export function BTCMapProvider({ children }) {
if (result.upserts?.length) await upsertPlaces(result.upserts);
}
// Aux providers (Bitcoin Jungle, MoneyBadger) — full snapshots streamed
// into SQLite in batches (see ingestProviderNDJSON). Only present when the
// weekly cadence was due; stamp the timestamp so the next 6 days skip them.
if (includeProviders && result.providers?.length) {
clearBTCMapClusterCache();
for (const provider of result.providers) {
await ingestProviderNDJSON(provider.source, provider.ndjson);
}
await setProviderLastSyncTime(Date.now());
}
await setLastModified(result.last_modified);
await setLastSyncTime(Date.now());
setDataVersion(v => v + 1);
@@ -94,26 +145,50 @@ export function BTCMapProvider({ children }) {
}
}, []);
// Fetch single place detail on demand (for merchant bottom sheet)
const getPlaceDetail = useCallback(async (placeId, privateKey, publicKey) => {
if (!privateKey || !publicKey) return null;
try {
const result = await fetchBackend(
'getBTCMapData',
{ action: 'detail', placeId },
privateKey,
publicKey,
);
return result || null;
} catch (err) {
console.log('[BTCMap] detail fetch error:', err);
return null;
}
}, []);
// Fetch single place detail on demand (for merchant bottom sheet).
// BTC Map has no detail stored locally → fetch from backend. Aux providers
// (bitcoinjungle/moneybadger) store all detail fields in SQLite → read local.
const getPlaceDetail = useCallback(
async (placeId, source, privateKey, publicKey) => {
try {
if (source && source !== 'btcmap') {
const row = await getProviderPlace(source, placeId);
if (!row) return null;
return {
id: row.native_id,
source: row.source,
name: row.name,
address: row.address,
lat: row.lat,
lon: row.lon,
icon: row.icon,
phone: row.phone,
website: row.website,
email: row.email,
lightning_address: row.lightning_address,
};
}
if (!privateKey || !publicKey) return null;
const result = await fetchBackend(
'getBTCMapData',
{ action: 'detail', placeId },
privateKey,
publicKey,
);
return result || null;
} catch (err) {
console.log('[BTCMap] detail fetch error:', err);
return null;
}
},
[],
);
const getPlacesInViewport = useCallback(
(minLat, maxLat, minLon, maxLon) =>
getPlacesInBbox(minLat, maxLat, minLon, maxLon),
async (minLat, maxLat, minLon, maxLon) => {
const rows = await getAllPlacesInBbox(minLat, maxLat, minLon, maxLon);
return dedupeMerge(rows);
},
[],
);
+1 -1
View File
@@ -2630,7 +2630,7 @@
"pay": "Bezahlen",
"lightning": "Lightning",
"lightningMessage": "Akzeptiert Bitcoin-Zahlungen sofort über das Lightning-Netzwerk",
"dataMessage": "Daten von btcmap.org"
"dataMessage": "Daten von {{source}}"
},
"filter": {
"categoriesTitle": "Kategorien",
+1 -1
View File
@@ -2630,7 +2630,7 @@
"pay": "Pay",
"lightning": "Lightning",
"lightningMessage": "Accepts bitcoin payments instantly over the Lightning Network",
"dataMessage": "Data from btcmap.org"
"dataMessage": "Data from {{source}}"
},
"filter": {
"categoriesTitle": "Categories",
+1 -1
View File
@@ -2630,7 +2630,7 @@
"pay": "Pagar",
"lightning": "Lightning",
"lightningMessage": "Acepta pagos en bitcoin instantáneamente a través de la red Lightning",
"dataMessage": "Datos de btcmap.org"
"dataMessage": "Datos de {{source}}"
},
"filter": {
"categoriesTitle": "Categorías",
+1 -1
View File
@@ -2630,7 +2630,7 @@
"pay": "Payer",
"lightning": "Lightning",
"lightningMessage": "Accepte les paiements en bitcoin instantanément via le réseau Lightning",
"dataMessage": "Données de btcmap.org"
"dataMessage": "Données de {{source}}"
},
"filter": {
"categoriesTitle": "Catégories",
+1 -1
View File
@@ -2630,7 +2630,7 @@
"pay": "Paga",
"lightning": "Lightning",
"lightningMessage": "Accetta pagamenti in bitcoin istantaneamente tramite la rete Lightning",
"dataMessage": "Dati da btcmap.org"
"dataMessage": "Dati da {{source}}"
},
"filter": {
"categoriesTitle": "Categorie",
+1 -1
View File
@@ -2629,7 +2629,7 @@
"pay": "Pagar",
"lightning": "Bitcoin",
"lightningMessage": "Aceita pagamentos Bitcoin",
"dataMessage": "Dados do btcmap.org"
"dataMessage": "Dados do {{source}}"
},
"filter": {
"categoriesTitle": "Categorias",
+1 -1
View File
@@ -2637,7 +2637,7 @@
"pay": "Оплатить",
"lightning": "Lightning",
"lightningMessage": "Принимает платежи в биткоинах мгновенно через сеть Lightning",
"dataMessage": "Данные с btcmap.org"
"dataMessage": "Данные с {{source}}"
},
"filter": {
"categoriesTitle": "Категории",
+1 -1
View File
@@ -2630,7 +2630,7 @@
"pay": "Betala",
"lightning": "Lightning",
"lightningMessage": "Tar emot bitcoinbetalningar direkt via Lightning-nätverket",
"dataMessage": "Data från btcmap.org"
"dataMessage": "Data från {{source}}"
},
"filter": {
"categoriesTitle": "Kategorier",