bgg-ottawa-sell-club/src/fetchCollection.js

import { load } from 'cheerio';
import fetch from 'node-fetch';
import DB from 'better-sqlite3';

const db_file = process.env.DATABASE_URL.replace('sqlite3:', '');
const db = DB(db_file);
db.pragma('journal_mode = WAL');

export async function fetch_guild_users(guild_id) {
    const res = await fetch(
        `https://boardgamegeek.com/xmlapi2/guild?members=1&id=` + guild_id,
    );

    if (res.status === 200) {
        const $page = load(await res.text());
        const users = [];
        $page('member').each(function (i, member) {
            users.push($page(this).attr('name'));
        });

        const insert = db.prepare(
            'INSERT OR IGNORE INTO bgg_user (username) VALUES(@username)',
        );

        users.forEach((username) => insert.run({ username }));

        return users;
    }

    throw new Error('sad');
}

export function extract_user_forsale(page) {
    const $ = load(page);

    const games = [];

    const data = $('item').map(function (i, item) {
        const data = {};
        data.bgg_id = $(this).attr('objectid');
        data.name = $(this).find('name').text();
        data.thumbnail = $(this).find('thumbnail').text();
        data.notes = $(this).find('conditiontext').text();
        const find_price = data.notes.match(/\$(\d+)/);
        data.price = undefined;
        if (find_price) data.price = parseInt(find_price[1]);

        games.push(data);
    });

    return games;
}

async function fetch_user_forsale(username, n = 1) {
    if (n > 5) throw new Error("couldn't get collection");

    const res = await fetch(
        `https://boardgamegeek.com/xmlapi2/collection?trade=1&username=${username}`,
    );

    if (res.status === 202) {
        return new Promise((accept, reject) => {
            setTimeout(() => {
                fetch_user_forsale(username, n + 1).then(accept);
            }, 2000);
        });
    }

    if (res.status === 200) return extract_user_forsale(await res.text());

    throw new Error("couldn't get the collection for " + username);
}

async function update_user_games(username) {
    const games = await fetch_user_forsale(username);

    const updated_at = new Date().toISOString();

    // TODO only change the games that changed
    // TODO only change users that got added or removed
    const insert = db.prepare(
        'INSERT into game (username, bgg_id, name, thumbnail, notes,price,updated_at) VALUES(@username,@bgg_id,@name,@thumbnail,@notes,@price,@updated_at)',
    );

    await db
        .prepare('DELETE FROM game where username = @username')
        .run({ username });

    for (let game of games) {
        console.log(game);
        insert.run({
            username,
            updated_at,
            ...game,
        });
    }
}
await update_user_games('gamingduo2')
    .then(() => console.log('is done'))
    .catch((e) => console.error(e));

/*
const usernames = await fetch_guild_users('1610');

await Promise.all(
    usernames.map( username => {
        return update_user_games(username).then( () => console.log(username, 'is done')).catch( e => console.error(e));
    })
)
*/