feat: add cached public news and free live data services

This commit is contained in:
Redacted-Coder 2026-09-08 18:34:07 -04:00 committed by GitHub
parent d62fd74573
commit 58aef7d617
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 314 additions and 0 deletions

62
server/cameraExpansion.js Normal file
View File

@ -0,0 +1,62 @@
export function publicVideoUrl(value) {
try {
const url = new URL(value);
return url.protocol === 'https:' && url.hostname === 'wzmedia.dot.ca.gov'
&& !url.username && !url.password && url.pathname.endsWith('.m3u8') ? url.href : '';
} catch { return ''; }
}
// Round-robin keeps a large early provider from excluding every later city.
export function balanceCameraCities(sources, limit, groupBy = source => source.city || source.provider || 'Other') {
const groups = new Map();
for (const source of sources) {
const key = groupBy(source);
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(source);
}
const result = [];
for (let i = 0; result.length < limit; i++) {
let added = false;
for (const group of groups.values()) {
if (group[i] && result.length < limit) { result.push(group[i]); added = true; }
}
if (!added) break;
}
return result;
}
const CITIES = [['Seattle', 47.6062, -122.3321], ['Tacoma', 47.2529, -122.4443],
['Olympia', 47.0379, -122.9007], ['Spokane', 47.6588, -117.4260],
['Vancouver WA', 45.628, -122.6739], ['Bellingham', 48.7519, -122.4787],
['Everett', 47.979, -122.202], ['Yakima', 46.6021, -120.5059],
['Tri-Cities', 46.2396, -119.1006], ['Wenatchee', 47.4235, -120.3103]];
export function normalizeWashingtonCameras(payload) {
return (payload?.features || []).flatMap(({ attributes: a = {}, geometry: g = {} }) => {
const lat = Number(g.y), lon = Number(g.x);
if (!Number.isFinite(lat) || !Number.isFinite(lon) || lat < 45 || lat > 50 || lon < -125 || lon > -116 || !a.OBJECTID) return [];
let image;
try {
image = new URL(a.ImageURL);
if (image.protocol !== 'https:' || image.username || image.password
|| !['images.wsdot.wa.gov', 'images.wsdot.com', 'www.wsdot.com', 'www.tripcheck.com'].includes(image.hostname)) return [];
} catch { return []; }
const nearest = [...CITIES].sort((a, b) => ((lat-a[1])**2 + ((lon-a[2])*0.68)**2) - ((lat-b[1])**2 + ((lon-b[2])*0.68)**2))[0];
return [{ id: `wsdot-${a.OBJECTID}`, name: String(a.CameraTitle || a.OBJECTID),
city: `${nearest[0]} area`, cityId: nearest[0].toLowerCase().replace(/\s/g, '-'),
provider: 'Washington State Department of Transportation', lat, lon,
headingDeg: ({ N: 0, E: 90, S: 180, W: 270 })[a.CompassDirection] ?? 0,
headingConfidence: 'low', pitchDeg: -18, fovDeg: 44, rangeM: 145,
mountHeightM: 8, groundElevationM: 0, feedType: 'image', url: image.href,
snapshotUrl: image.href, sourceKind: 'wsdot-open-data', license: 'WSDOT public traffic cameras; provider snapshots' }];
});
}
export async function loadWashingtonCameras() {
if (process.env.CCTV_WSDOT_ENABLED === '0') return [];
try {
const url = 'https://data.wsdot.wa.gov/arcgis/rest/services/TravelInformation/TravelInfoCamerasWeather/FeatureServer/0/query?where=1%3D1&outFields=OBJECTID,CameraTitle,ImageURL,CompassDirection&outSR=4326&f=json&resultRecordCount=2000';
const r = await fetch(url, { signal: AbortSignal.timeout(15000) });
if (!r.ok) throw new Error(`HTTP ${r.status}`);
return balanceCameraCities(normalizeWashingtonCameras(await r.json()), 300);
} catch { console.warn('[CCTV] Washington camera catalog unavailable'); return []; }
}

View File

@ -0,0 +1,21 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { publicVideoUrl, balanceCameraCities, normalizeWashingtonCameras } from './cameraExpansion.js';
test('only official HTTPS HLS links reach the browser player', () => {
assert.equal(publicVideoUrl('https://wzmedia.dot.ca.gov/D7/cam/playlist.m3u8'), 'https://wzmedia.dot.ca.gov/D7/cam/playlist.m3u8');
for (const url of ['https://evil.test/a.m3u8', 'http://wzmedia.dot.ca.gov/a.m3u8', 'https://key@wzmedia.dot.ca.gov/a.m3u8', 'https://wzmedia.dot.ca.gov.evil.test/a.m3u8']) assert.equal(publicVideoUrl(url), '');
});
test('city balancing does not let a large provider crowd out later cities', () => {
const input = [{city:'A',id:1},{city:'A',id:2},{city:'A',id:3},{city:'B',id:4},{city:'C',id:5}];
assert.deepEqual(balanceCameraCities(input, 3).map(x=>x.id), [1,4,5]);
assert.equal(balanceCameraCities(input, 20).length, 5);
assert.deepEqual(balanceCameraCities([], 20), []);
});
test('Washington rejects invalid coordinates and untrusted image origins', () => {
const feature = { attributes: { OBJECTID: 1, CameraTitle: 'I-5', ImageURL: 'https://images.wsdot.wa.gov/a.jpg' }, geometry: {x:-122.33,y:47.60} };
const [camera] = normalizeWashingtonCameras({features:[feature]});
assert.equal(camera.city, 'Seattle area');
assert.equal(camera.feedType, 'image');
assert.equal(normalizeWashingtonCameras({features:[{...feature,geometry:{x:0,y:0}}]}).length,0);
assert.equal(normalizeWashingtonCameras({features:[{...feature,attributes:{...feature.attributes,ImageURL:'http://localhost/private'}}]}).length,0);
});

65
server/freeServices.js Normal file
View File

@ -0,0 +1,65 @@
import { normalizePhotonPlace } from '../src/freeGeocode.js';
export function freeProviderConfig(env = process.env) {
const hasKey = key => Boolean(String(env[key] || '').trim());
return {
freeOnly: env.GEV_FREE_ONLY === '1',
ships: hasKey('AISSTREAM_API_KEY'),
fires: hasKey('FIRMS_MAP_KEY'),
traffic: hasKey('TOMTOM_API_KEY'),
imagery: hasKey('CESIUM_ION_TOKEN'),
voice: env.GEV_FREE_ONLY !== '1' && hasKey('OPENAI_API_KEY'),
google: env.GEV_FREE_ONLY !== '1' && hasKey('GOOGLE_MAPS_API_KEY'),
};
}
/** One bounded, cached search on Enter; no background autocomplete traffic. */
export function freeServicesPlugin({ fetchJson } = {}) {
const cache = new Map();
const inFlight = new Map();
let lastRequest = 0;
let queue = Promise.resolve();
const send = (res, status, payload) => {
res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
res.end(JSON.stringify(payload));
};
const install = middlewares => {
middlewares.use('/api/free-providers', (req, res) => {
if (req.method !== 'GET') return send(res, 405, { error: 'Method not allowed' });
send(res, 200, freeProviderConfig());
});
middlewares.use('/api/free-geocode', async (req, res) => {
if (req.method !== 'GET') return send(res, 405, { error: 'Method not allowed' });
const q = new URL(req.url || '', 'http://localhost').searchParams.get('q')?.trim();
if (!q || q.length > 200) return send(res, 400, { error: 'Enter a place name (1200 characters).' });
const key = q.toLowerCase();
const cached = cache.get(key);
if (cached && Date.now() - cached.at < 24 * 60 * 60_000) return send(res, 200, cached.payload);
if (!inFlight.has(key)) {
if (inFlight.size >= 4) return send(res, 429, { error: 'Search busy; retry shortly.' });
const task = queue.then(async () => {
const delay = Math.max(0, 1100 - (Date.now() - lastRequest));
if (delay) await new Promise(resolve => setTimeout(resolve, delay));
lastRequest = Date.now();
const url = new URL(process.env.GEV_PHOTON_URL || 'https://photon.komoot.io/api/');
url.search = new URLSearchParams({ q, limit: '1', lang: 'en' }).toString();
const data = await fetchJson(url.href, {
timeoutMs: 9000, maxBytes: 128 * 1024,
headers: { 'User-Agent': 'GodsEyeView/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)' },
});
if (!Array.isArray(data?.features)) throw new Error('Invalid geocoder response');
const payload = { place: normalizePhotonPlace(data.features[0]) };
cache.set(key, { payload, at: Date.now() });
while (cache.size > 200) cache.delete(cache.keys().next().value);
return payload;
});
queue = task.catch(() => {});
inFlight.set(key, task);
task.finally(() => inFlight.delete(key)).catch(() => {});
}
try { send(res, 200, await inFlight.get(key)); }
catch { send(res, 503, { error: 'Place search is temporarily unavailable.' }); }
});
};
return { name: 'free-services', configureServer: s => install(s.middlewares), configurePreviewServer: s => install(s.middlewares) };
}

76
server/liveViews.js Normal file
View File

@ -0,0 +1,76 @@
import { WORLD_PLACES } from '../src/worldPlaces.js';
export const validPoint = (lat,lon) => Number.isFinite(lat) && Number.isFinite(lon) && Math.abs(lat)<=90 && Math.abs(lon)<=180;
export function normalizeDisasters(data) {
if (!Array.isArray(data?.features)) throw new Error('Invalid disaster feed');
return data.features.flatMap(f => {
let c = f.geometry?.coordinates; if (Array.isArray(c?.[0])) c=c[0];
const p=f.properties||{};
if (f.geometry?.type!=='Point' || !validPoint(c?.[1],c?.[0])) return [];
return [{id:`${p.eventtype}-${p.eventid}`,lat:c[1],lon:c[0],name:String(p.title||'Disaster report'),
type:p.eventtype,level:p.alertlevel,time:p.todate,description:String(p.description||'').slice(0,1500)}];
}).filter(x=>['TC','FL','DR','WF','VO'].includes(x.type))
.sort((a,b)=>Number(a.type==='WF')-Number(b.type==='WF'))
.slice(0,300);
}
export function normalizeRadar(raw) {
if(raw?.host!=='https://tilecache.rainviewer.com'||!Array.isArray(raw?.radar?.past))throw new Error('Invalid radar feed');
const frames=raw.radar.past.filter(f=>Number.isInteger(f.time)&&f.time>0&&/^\/v2\/radar\/[a-zA-Z0-9_-]{1,64}$/.test(f.path))
.sort((a,b)=>a.time-b.time).slice(-13).map(({time,path})=>({time,path}));
if(!frames.length)throw new Error('Empty radar feed');
return {source:'RainViewer',host:raw.host,frames};
}
export function normalizeTransit(data) {
if (!Array.isArray(data?.data)) throw new Error('Invalid transit feed');
return data.data.flatMap(v => {
const a=v.attributes||{};
if (!validPoint(a.latitude,a.longitude)) return [];
return [{id:v.id,lat:a.latitude,lon:a.longitude,name:`${v.relationships?.route?.data?.id||'Transit'} · ${a.label||v.id}`,
time:a.updated_at,status:a.current_status}];
}).slice(0,200);
}
export function liveViewsPlugin({fetchJson}) {
const cache=new Map(), pending=new Map();
async function obtain(key,ttl,job) {
const old=cache.get(key);
if(old && Date.now()-old.receivedAt<ttl) return {...old,stale:false};
if(pending.has(key)) return pending.get(key);
if(pending.size>=4) throw new Error('Busy');
const task=(async()=>{
try {
const result={...await job(),receivedAt:Date.now()};cache.set(key,result);
while(cache.size>80) cache.delete(cache.keys().next().value);
return {...result,stale:false};
} catch(e) {
if(old && Date.now()-old.receivedAt<3600000) return {...old,stale:true};
throw e;
} finally {pending.delete(key);}
})();
pending.set(key,task); return task;
}
const install=middlewares=>middlewares.use('/api/live-views',async(req,res)=>{
const send=(status,body)=>{res.writeHead(status,{'Content-Type':'application/json','Cache-Control':'no-store'});res.end(JSON.stringify(body));};
if(req.method!=='GET') return send(405,{error:'GET only'});
const u=new URL(req.url||'/','http://localhost');
try {
if(u.pathname==='/radar') return send(200,await obtain('radar',300000,async()=>normalizeRadar(await fetchJson('https://api.rainviewer.com/public/weather-maps.json',{timeoutMs:15000,maxBytes:100000}))));
if(u.pathname==='/disasters') return send(200,await obtain('disasters',360000,async()=>({source:'GDACS',kind:'Published disaster reports · up to 300; cyclone/flood/drought/volcano reports prioritized over wildfire overflow',items:normalizeDisasters(await fetchJson('https://www.gdacs.org/contentdata/xml/gdacsAPP_Home.geojson',{timeoutMs:15000,maxBytes:2000000}))})));
if(u.pathname==='/transit') return send(200,await obtain('transit',30000,async()=>({source:'MBTA',kind:'Reported vehicle positions · Boston only · up to 200 vehicles',items:normalizeTransit(await fetchJson('https://api-v3.mbta.com/vehicles?page%5Blimit%5D=200',{timeoutMs:15000,maxBytes:2000000}))})));
if(u.pathname==='/weather') {
const world=u.searchParams.get('world')==='1';
const lat=Number(u.searchParams.get('lat')),lon=Number(u.searchParams.get('lon'));
if(!world && (!u.searchParams.has('lat')||!u.searchParams.has('lon')||!validPoint(lat,lon))) return send(400,{error:'Valid coordinates required'});
const points=world?WORLD_PLACES:[{name:'Selected location',lat:Math.round(lat*100)/100,lon:Math.round(lon*100)/100}];
const key=world?'weather-world':`weather-${points[0].lat}-${points[0].lon}`;
return send(200,await obtain(key,600000,async()=>{
const q=new URLSearchParams({latitude:points.map(p=>p.lat).join(','),longitude:points.map(p=>p.lon).join(','),current:'temperature_2m,apparent_temperature,precipitation,weather_code,wind_speed_10m',daily:'temperature_2m_max,temperature_2m_min,precipitation_probability_max',forecast_days:'3',timezone:'GMT'});
const raw=await fetchJson(`https://api.open-meteo.com/v1/forecast?${q}`,{timeoutMs:15000,maxBytes:1000000});
const rows=Array.isArray(raw)?raw:[raw];
if(rows.length!==points.length||rows.some(r=>!r.current)) throw new Error('Incomplete weather');
return {source:'Open-Meteo · CC BY 4.0',kind:'Model-based current conditions and forecasts · not station observations',items:rows.map((r,i)=>({...points[i],current:r.current,daily:r.daily,units:r.current_units}))};
}));
}
send(404,{error:'Unknown feed'});
}catch{send(503,{error:'Provider unavailable. Retry later; no empty success substituted.'});}
});
return {name:'live-views',configureServer:s=>{install(s.middlewares);},configurePreviewServer:s=>{install(s.middlewares);}};
}

50
server/liveViews.test.mjs Normal file
View File

@ -0,0 +1,50 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {normalizeDisasters,normalizeTransit,liveViewsPlugin} from './liveViews.js';
test('disaster parser accepts published nested points and rejects invalid coordinates',()=>{
const feature={geometry:{type:'Point',coordinates:[[10,20]]},properties:{eventid:1,eventtype:'TC',title:'Storm'}};
assert.equal(normalizeDisasters({features:[feature]})[0].lat,20);
assert.equal(normalizeDisasters({features:[{...feature,geometry:{type:'Point',coordinates:[10,100]}}]}).length,0);
assert.throws(()=>normalizeDisasters({}));
});
test('transit retains observation timestamp and rejects missing positions',()=>{
const data={data:[{id:'1',attributes:{latitude:42,longitude:-71,updated_at:'2026-09-06T00:00:00Z'}},{id:'2',attributes:{}}]};
const out=normalizeTransit(data);assert.equal(out.length,1);assert.equal(out[0].time,'2026-09-06T00:00:00Z');
});
function harness(fetchJson){let handler;liveViewsPlugin({fetchJson}).configureServer({middlewares:{use:(_,fn)=>handler=fn}});return(url,method='GET')=>new Promise(resolve=>{let status;handler({url,method},{writeHead:s=>status=s,end:s=>resolve({status,body:JSON.parse(s)})});});}
test('requests are validated, coalesced and cached without exposing raw failures',async()=>{
let calls=0;const request=harness(async()=>{calls++;await new Promise(r=>setTimeout(r,5));return{data:[]};});
assert.equal((await request('/weather?lat=999&lon=0')).status,400);
assert.equal((await request('/weather')).status,400);
assert.equal((await request('/transit','POST')).status,405);
const results=await Promise.all([request('/transit'),request('/transit')]);
assert.equal(calls,1);assert.equal(results[0].status,200);assert.equal(results[0].body.stale,false);
await request('/transit');assert.equal(calls,1);
const bad=harness(async()=>{throw new Error('secret upstream key');});
const failure=await bad('/transit');assert.equal(failure.status,503);assert.ok(!JSON.stringify(failure).includes('secret'));
});
test('Vite setup returns no accidental post-install middleware hook',()=>{
const connect=()=>{};
const plugin=liveViewsPlugin({fetchJson:async()=>({})});
assert.equal(plugin.configureServer({middlewares:{use:()=>connect}}),undefined);
assert.equal(plugin.configurePreviewServer({middlewares:{use:()=>connect}}),undefined);
});
test('expired cache survives provider outage only with an explicit stale flag',async()=>{
const realNow=Date.now;let now=100000,fail=false;
Date.now=()=>now;
try {
const request=harness(async()=>{if(fail)throw new Error('offline');return{data:[]};});
const first=await request('/transit');assert.equal(first.body.stale,false);
now+=31000;fail=true;
const old=await request('/transit');assert.equal(old.status,200);assert.equal(old.body.stale,true);assert.equal(old.body.receivedAt,100000);
now+=3600001;assert.equal((await request('/transit')).status,503);
}finally{Date.now=realNow;}
});
test('radar accepts only the documented host and frame paths',async()=>{
const {normalizeRadar}=await import('./liveViews.js');
const raw={host:'https://tilecache.rainviewer.com',radar:{past:[{time:100,path:'/v2/radar/100'}]}};
assert.equal(normalizeRadar(raw).frames.length,1);
assert.throws(()=>normalizeRadar({...raw,host:'https://example.com'}));
assert.throws(()=>normalizeRadar({...raw,radar:{past:[{time:100,path:'/v2/radar/../200'}]}}));
const request=harness(async()=>raw);assert.equal((await request('/radar')).status,200);
});

31
server/situationNews.js Normal file
View File

@ -0,0 +1,31 @@
import {REGIONS,TOPICS,cleanArticles} from '../src/situationModel.js';
export function situationNewsPlugin({fetchText,parseArticles}){
const cache=new Map(),pending=new Map();
async function obtain(region,topic,hours){
const key=region.id+':'+topic+':'+hours,old=cache.get(key);
if(old&&Date.now()-old.receivedAt<300000)return {...old,stale:false};
if(pending.has(key))return pending.get(key);
if(pending.size>=4)throw new Error('Busy');
const task=(async()=>{
try{
const query=[region.query,TOPICS[topic].query,'when:'+hours+'h'].filter(Boolean).join(' ');
const params=new URLSearchParams({q:query,hl:'en-US',gl:'US',ceid:'US:en'});
const xml=await fetchText('https://news.google.com/rss/search?'+params,{timeoutMs:12000,maxBytes:1000000});
// A valid empty channel is distinct from an upstream error page.
if(!/<rss[\s>]/i.test(xml)||!/<channel[\s>]/i.test(xml))throw new Error('Invalid feed');
const result={region:region.id,topic,hours,receivedAt:Date.now(),source:'Google News RSS · publisher headlines',items:cleanArticles(parseArticles(xml,100),Date.now(),hours)};
cache.set(key,result);while(cache.size>48)cache.delete(cache.keys().next().value);
return {...result,stale:false};
}catch(e){if(old&&Date.now()-old.receivedAt<3600000)return {...old,stale:true};throw e;}
finally{pending.delete(key);}
})();pending.set(key,task);return task;
}
const install=m=>m.use('/api/situation-news',async(req,res)=>{
const send=(code,body)=>{res.writeHead(code,{'Content-Type':'application/json','Cache-Control':'no-store'});res.end(JSON.stringify(body));};
if(req.method!=='GET')return send(405,{error:'GET only'});
const u=new URL(req.url||'/','http://localhost'),region=REGIONS.find(r=>r.id===(u.searchParams.get('region')||'world')),topic=u.searchParams.get('topic')||'conflict',hours=Number(u.searchParams.get('hours')||24);
if(!region||!Object.hasOwn(TOPICS,topic)||![6,24,48].includes(hours))return send(400,{error:'Choose a supported region, topic and time window.'});
try{send(200,await obtain(region,topic,hours));}catch{send(503,{error:'News provider unavailable; retry later.'});}
});
return {name:'situation-news',configureServer:s=>{install(s.middlewares);},configurePreviewServer:s=>{install(s.middlewares);}};
}

View File

@ -0,0 +1,9 @@
import {test} from 'node:test';import assert from 'node:assert/strict';
import {situationNewsPlugin} from './situationNews.js';
function harness(fetchText){let handler;situationNewsPlugin({fetchText,parseArticles:()=>[]}).configureServer({middlewares:{use:(_,f)=>handler=f}});return url=>new Promise(resolve=>{let code;handler({method:'GET',url},{writeHead:value=>{code=value;},end:s=>resolve({code,body:JSON.parse(s)})});});}
test('news validates choices, coalesces requests and rejects upstream error pages',async()=>{
let calls=0;const request=harness(async()=>{calls++;await new Promise(r=>setTimeout(r,5));return '<rss><channel></channel></rss>';});
assert.equal((await request('/?region=unknown')).code,400);
const out=await Promise.all([request('/'),request('/')]);assert.equal(calls,1);assert.equal(out[0].body.stale,false);
const bad=harness(async()=>'<html>error</html>');assert.equal((await bad('/')).code,503);
});