bgg-ottawa-sell-club/src/fetchCollection.js

import { load } from 'cheerio';
import fetch from 'node-fetch';
import DB from 'better-sqlite3';

const db_file = process.env.DATABASE_URL.replace('sqlite3:','');
const db = DB(db_file);
db.pragma('journal_mode = WAL');

export async function fetch_guild_users (guild_id) {
    const res = await fetch(`https://boardgamegeek.com/xmlapi2/guild?members=1&id=`+guild_id);

    if( res.status === 200 ) {
        const $page = load(await res.text());
        const users = [];
        $page('member').each( function(i,member) {
            users.push(       $page(this).attr('name'));
        });

        const insert = db.prepare( 'INSERT OR IGNORE INTO bgg_user (username) VALUES(@username)' );

        users.forEach( username => insert.run({username}));

        return users;
    }

    throw new Error('sad');
}

export function extract_user_forsale(page) {

    const $ = load(page);

    const games = [];

    const data = $('item').map( function (i,item) {
        const data = {};
        data.bgg_id = $(this).attr('objectid');
        data.name = $(this).find('name').text();
        data.thumbnail = $(this).find('thumbnail').text();
        data.notes = $(this).find('conditiontext').text();
        const find_price = data.notes.match(/\$(\d+)/);  
        data.price = undefined;
        if(find_price ) data.price = parseInt(find_price[1]);

        games.push(data);
    })

    return games;
}

async function fetch_user_forsale(username,n=1) {
    if(n>5) throw new Error("couldn't get collection");

    const res = await fetch(`https://boardgamegeek.com/xmlapi2/collection?trade=1&username=${username}`);

    if ( res.status === 202 ) {
        return new Promise( (accept,reject) => {
            setTimeout(()=> {
                fetch_user_forsale(username,n+1).then(accept)
            }, 2000)
        })
    }

    if( res.status === 200 )
        return extract_user_forsale( await res.text() );

    throw new Error("couldn't get the collection for " + username);
}

async function update_user_games(username) {
    const games = await fetch_user_forsale(username);

    // TODO only change the games that changed 
    // TODO only change users that got added or removed
    const insert = db.prepare( 'INSERT into game (username, bgg_id, name, thumbnail, notes,price) VALUES(@username,@bgg_id,@name,@thumbnail,@notes,@price)' );

    await db.prepare('DELETE FROM game where username = @username').run({username});

    for ( let game of games ) {
        console.log(game);
        insert.run({
            username,
            ...game
        });
    }


}
await update_user_games('yenzie').then( () => console.log('is done')).catch( e => console.error(e));

/*
const usernames = await fetch_guild_users('1610');

await Promise.all(
    usernames.map( username => {
        return update_user_games(username).then( () => console.log(username, 'is done')).catch( e => console.error(e));
    })
)
*/