import { load } from 'cheerio'; import fetch from 'node-fetch'; import DB from 'better-sqlite3'; const db_file = process.env.DATABASE_URL.replace('sqlite3:',''); const db = DB(db_file); db.pragma('journal_mode = WAL'); export async function fetch_guild_users (guild_id) { const res = await fetch(`https://boardgamegeek.com/xmlapi2/guild?members=1&id=`+guild_id); if( res.status === 200 ) { const $page = load(await res.text()); const users = []; $page('member').each( function(i,member) { users.push( $page(this).attr('name')); }); const insert = db.prepare( 'INSERT OR IGNORE INTO bgg_user (username) VALUES(@username)' ); users.forEach( username => insert.run({username})); return users; } throw new Error('sad'); } export function extract_user_forsale(page) { const $ = load(page); const games = []; const data = $('item').map( function (i,item) { const data = {}; data.bgg_id = $(this).attr('objectid'); data.name = $(this).find('name').text(); data.thumbnail = $(this).find('thumbnail').text(); data.notes = $(this).find('conditiontext').text(); const find_price = data.notes.match(/\$(\d+)/); data.price = undefined; if(find_price ) data.price = parseInt(find_price[1]); games.push(data); }) return games; } async function fetch_user_forsale(username,n=1) { if(n>5) throw new Error("couldn't get collection"); const res = await fetch(`https://boardgamegeek.com/xmlapi2/collection?trade=1&username=${username}`); if ( res.status === 202 ) { return new Promise( (accept,reject) => { setTimeout(()=> { fetch_user_forsale(username,n+1).then(accept) }, 2000) }) } if( res.status === 200 ) return extract_user_forsale( await res.text() ); throw new Error("couldn't get the collection for " + username); } async function update_user_games(username) { const games = await fetch_user_forsale(username); // TODO only change the games that changed // TODO only change users that got added or removed const insert = db.prepare( 'INSERT into game (username, bgg_id, name, thumbnail, notes,price) VALUES(@username,@bgg_id,@name,@thumbnail,@notes,@price)' ); await db.prepare('DELETE FROM game where username = @username').run({username}); for ( let game of games ) { console.log(game); insert.run({ username, ...game }); } } await update_user_games('yenzie').then( () => console.log('is done')).catch( e => console.error(e)); /* const usernames = await fetch_guild_users('1610'); await Promise.all( usernames.map( username => { return update_user_games(username).then( () => console.log(username, 'is done')).catch( e => console.error(e)); }) ) */