software/hoardy-web/./extension/lib/util.js

Passively capture, archive, and hoard your web browsing history, including the contents of the pages you visit, for later offline viewing, replay, mirroring, data scraping, and/or indexing. Your own personal private Wayback Machine that can also archive HTTP POST requests and responses, as well as most other HTTP-level data.

Files

Raw Source

Contents

/*
 * Copyright (c) 2024-2026 Jan Malakhovski <oxij@oxij.org>
 *
 * This file is a part of `hoardy-web` project.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Some utility functions and constants specific to `Hoardy-Web`.
 *
 * This file exists to prevent duplication between the core and the UI
 * parts of `Hoardy-Web`.
 *
 */

"use strict";

function pushFirstTo(archivables, res) {
    for (let [v, _x] of archivables) {
        res.push(v);
    }
    return res;
}

// see https://en.wikipedia.org/wiki/HTTP_status_codes
// and https://datatracker.ietf.org/doc/html/rfc9110
let redirectStatusCodes = new Set([301, 302, 303, 307, 308]);
let transientStatusCodes = new Set([
    401, 402, 403, 404, 407, 408, 409,
    //
    412, 416, 418, 421, 423, 424, 425,
    //
    426, 429, 451,
    //
    500, 502, 503, 504, 507, 511,
    // unofficial ones
    419, 440, 450, 495, 496,
    //
    509, 520, 521, 522, 523, 524, 525,
    //
    526, 530, 540, 598, 599,
]);

// internal URLs
let selfURL = browser.runtime.getURL("/");
let popupPageURL = browser.runtime.getURL("/page/popup.html");
let changelogPageURL = browser.runtime.getURL("/page/changelog.html");
let helpPageURL = browser.runtime.getURL("/page/help.html");
let statePageURL = browser.runtime.getURL("/page/state.html");
let savedPageURL = browser.runtime.getURL("/page/saved.html");

function iconPath(name, size) {
    if (useSVGIcons) {
        return `/icon/${name}.svg?v=${manifest.version}`;
    }
    return `/icon/${size}/${name}.png?v=${manifest.version}`;
}

function iconURL(name, size) {
    return browser.runtime.getURL(iconPath(name, size));
}

function mkIcons(what) {
    return {
        128: iconPath(what, 128),
    };
}

async function getTabs(query) {
    let tabs;
    let specific = false;

    if (Array.isArray(query) && query.length === 1) {
        query = query[0];
    }

    if (query === null) {
        tabs = await browser.tabs.query({});
    } else if (typeof query === "number") {
        let res = await browser.tabs.get(query);
        tabs = [res];
        specific = true;
    } else if (Array.isArray(query)) {
        let set = new Set(query);
        let res = await browser.tabs.query({});
        tabs = res.filter((tab) => set.has(tab.id));
        specific = true;
    } else if (query.windowId !== undefined) {
        let windowId = query.windowId;
        if (windowId === true) {
            tabs = await browser.tabs.query({ currentWindow: true });
        } else {
            tabs = await browser.tabs.query({ windowId });
        }
    } else {
        throw new Error("bad query");
    }

    return [tabs, specific];
}

function getStateTabIdOrTabId(tab) {
    return getMapURLParam(statePageURL, "tab", getTabURL(tab), toNumber, tab.id, tab.id);
}

function showChangelog(...args) {
    return showInternalPageAtNode(changelogPageURL, ...args);
}

function showHelp(...args) {
    return showInternalPageAtNode(helpPageURL, ...args);
}

function showState(sessionId, windowId, tabId, ...args) {
    let nargs = [
        ["session", sessionId],
        ["window", windowId],
        ["tab", tabId],
    ];
    nargs = nargs.filter((a) => a[1] !== null);
    let parts = nargs.map((a) => `${a[0]}=${a[1]}`);
    let query = parts.join("&");
    let q = query !== "" ? "?" : "";
    return showInternalPageAtNode(statePageURL + q + query, ...args);
}

function showSaved(...args) {
    return showInternalPageAtNode(savedPageURL, ...args);
}

function broadcastToPopup(...args) {
    return broadcastToName(true, "popup", ...args);
}

function broadcastToHelp(...args) {
    return broadcastToName(true, "help", ...args);
}

function broadcastToState(tabId, ...args) {
    if (tabId === undefined) {
        // nothing to do
        return;
    }

    if (tabId === null) {
        // broadcast to all
        return broadcastToNamePrefix(true, "state", ...args);
    }

    // broadcast to per-tab pages
    let [lazy, res] = broadcastToNamePrefix(true, `state#${tabId}`, ...args);
    // and to the global ones
    return broadcastToName(lazy, "state", ...res);
}

function broadcastToSaved(...args) {
    return broadcastToName(true, "saved", ...args);
}

function setPageState(state) {
    document.getElementById("body").style.display = state === "done" ? "block" : "none";
    document.getElementById("body_loading").style.display = state === "loading" ? "block" : "none";
    document.getElementById("body_error").style.display = state === "error" ? "block" : "none";
}

function setPageLoading() {
    resetSingletonTimeout(
        scheduledUI,
        "setPage",
        300,
        () => {
            replaceElements(document.getElementById("body_loading"), "p", "Loading...");
            setPageState("loading");
        },
        100,
        true,
    );
}

function setPageSettling() {
    resetSingletonTimeout(
        scheduledUI,
        "setPage",
        300,
        () => {
            replaceElements(
                document.getElementById("body_loading"),
                "p",
                "Waiting for the core to settle...",
            );
            setPageState("loading");
        },
        100,
        true,
    );
}

function setPageError(error) {
    logError(error);

    resetSingletonTimeout(
        scheduledUI,
        "setPage",
        0,
        () => {
            replaceElements(document.getElementById("body_error"), [
                ["h1", "Exception"],
                ["pre", "code", errorMessageOf(error)],
                ["h2", "To see more details"],
                [
                    "ul",
                    [
                        [
                            "li",
                            [
                                ["p", "On a Firefox-based browser, go to"],
                                ["pre", "code", "about:debugging#/runtime/this-firefox"],
                                [
                                    "p",
                                    'Then, click "Inspect" button on "Hoardy-Web", select "Console".',
                                ],
                            ],
                        ],
                        [
                            "li",
                            [
                                ["p", "On a Chromium-based browser, go to"],
                                ["pre", "code", "chrome://extensions/"],
                                [
                                    "p",
                                    'Then, click "Inspect views" link on "Hoardy-Web", select "Console".',
                                ],
                            ],
                        ],
                    ],
                ],
            ]);
            setPageState("error");
        },
        0,
        true,
    );
}

function setPageDone() {
    resetSingletonTimeout(
        scheduledUI,
        "setPage",
        0,
        () => {
            setPageState("done");
        },
        0,
        true,
    );
}

function setRootClasses(config) {
    let sparse = config.sparse;
    if (sparse === null && isMobile) {
        sparse = true;
    }

    let dark = config.colors;
    if (dark === null && window.matchMedia("(prefers-color-scheme: dark)").matches) {
        dark = true;
    }

    let dnow = new Date();
    let dm = dnow.getMonth() + 1; // JavaScript is ridiculous
    let dd = dnow.getDate();

    let season = config.season;
    function can(name) {
        return (
            config.seasonal &&
            season[name] !== false &&
            Array.from(Object.keys(season)).every((k) => k === name || season[k] !== true)
        );
    }

    let halloween =
        can("halloween") &&
        ((dm === 10 && dd >= 30) || (dm === 11 && dd <= 1) || season.halloween === true);
    let winter =
        can("winter") &&
        ((dm === 12 && dd >= 20) || (dm === 1 && dd <= 8) || season.winter === true);

    if (halloween || winter) {
        dark = true;
    }

    let droot = getRootNode(document);

    setConditionalClass(droot, "sparse", sparse);

    setConditionalClass(droot, "light", !dark);
    setConditionalClass(droot, "colorblind", config.colorblind);

    setConditionalClass(droot, "season", halloween || winter);
    setConditionalClass(droot, "halloween", halloween);
    setConditionalClass(droot, "winter", winter);

    return droot;
}

function mapShortcutName(func, name) {
    let children;
    if (name.startsWith("toggleTabConfigChildren")) {
        name = name.substr(23);
        children = true;
    } else {
        name = name.substr(15);
        children = false;
    }
    name = uncapitalize(name);
    // TODO: remove
    if (name === "tracking") {
        name = "collecting";
    }
    return func(name, children);
}

function isUnknownError(error) {
    if (!useDebugger) {
        // Firefox
        if (
            error === "webRequest::NS_ERROR_ABORT" ||
            error === "webRequest::NS_BINDING_ABORTED" ||
            error === "webRequest::NS_ERROR_NET_ON_WAITING_FOR" ||
            error === "webRequest::NS_ERROR_NET_ON_RESOLVED" ||
            error === "webRequest::NS_ERROR_UNKNOWN_HOST" ||
            error === "webRequest::NS_ERROR_NET_ON_SENDING_TO" ||
            error === "webRequest::NS_ERROR_NET_PARTIAL_TRANSFER" ||
            error === "webRequest::NS_ERROR_UNEXPECTED" ||
            error === "webRequest::NS_IMAGELIB_ERROR_FAILURE" ||
            error === "webRequest::capture::EMIT_FORCED::BY_CLOSED_TAB" ||
            error === "webRequest::capture::EMIT_FORCED::BY_USER" ||
            error === "filterResponseData::Channel redirected"
        ) {
            return false;
        }
    } else {
        // Chromium
        // eslint-disable-next-line no-lonely-if
        if (
            error === "webRequest::net::ERR_ABORTED" ||
            error === "webRequest::net::ERR_CANCELED" ||
            error === "webRequest::net::ERR_FAILED" ||
            error === "webRequest::net::ERR_BLOCKED_BY_CLIENT" ||
            error === "webRequest::net::ERR_CONNECTION_CLOSED" ||
            error === "webRequest::capture::CANCELED::NO_DEBUGGER" ||
            error === "webRequest::capture::EMIT_FORCED::BY_CLOSED_TAB" ||
            error === "webRequest::capture::EMIT_FORCED::BY_DETACHED_DEBUGGER" ||
            error === "webRequest::capture::EMIT_FORCED::BY_USER" ||
            error === "debugger::net::ERR_ABORTED" ||
            error === "debugger::net::ERR_CANCELED" ||
            error === "debugger::net::ERR_FAILED" ||
            error === "debugger::net::ERR_BLOCKED_BY_CLIENT" ||
            error === "debugger::net::ERR_CONNECTION_CLOSED" ||
            error === "debugger::capture::EMIT_FORCED::BY_CLOSED_TAB" ||
            error === "debugger::capture::EMIT_FORCED::BY_DETACHED_DEBUGGER" ||
            error === "debugger::capture::EMIT_FORCED::BY_USER" ||
            error === "debugger::capture::NO_RESPONSE_BODY::DETACHED_DEBUGGER" ||
            error === "debugger::capture::NO_RESPONSE_BODY::ACCESS_DENIED" ||
            error === "debugger::capture::NO_RESPONSE_BODY::OTHER" ||
            error.startsWith("debugger::net::ERR_BLOCKED::")
        ) {
            return false;
        }
    }
    return true;
}

function isIncompleteError(error) {
    if (!useDebugger) {
        // Firefox
        if (
            error === "webRequest::NS_ERROR_ABORT" ||
            error === "webRequest::NS_BINDING_ABORTED" ||
            error === "webRequest::NS_ERROR_NET_ON_SENDING_TO" ||
            error === "webRequest::NS_ERROR_NET_PARTIAL_TRANSFER" ||
            error === "webRequest::NS_ERROR_UNEXPECTED"
        ) {
            return true;
        }
    }
    return false;
}

function isImportantError(error) {
    if (
        (error.startsWith("webRequest::capture::") &&
            error !== "webRequest::capture::CANCELED::BY_WORK_OFFLINE") ||
        error.startsWith("debugger::capture::")
    ) {
        return true;
    }
    return false;
}

function isTrivialError(error) {
    if (!useDebugger) {
        // Firefox
        if (error === "filterResponseData::Channel redirected") {
            return true;
        }
    }
    return false;
}

// Merge two `updatedTabId`s, `undefined` meanse "none", and `null` means `all`.
function mergeUpdatedTabIds(a, b) {
    if (a === b) {
        return a;
    }
    if (a === undefined) {
        return b;
    }
    if (b === undefined) {
        return a;
    }
    return null;
}

// archival status of a loggable

const archivedViaExportAs = 1;
const archivedViaSubmitHTTP = 2;
// NB: this one does not get written out to reqres, it's only used for `wantArchived` checks in
// `archive` function
const archivedIntoLS = 4;

function newRearchiveVars() {
    return {
        andDelete: false,
        andRewrite: false,
    };
}

function updateRearchiveVars(rearchive, path) {
    switch (path) {
        case "rearchive.andDelete":
            rearchive.andRewrite = rearchive.andRewrite && !rearchive.andDelete;
            break;
        case "rearchive.andRewrite":
            rearchive.andDelete = rearchive.andDelete && !rearchive.andRewrite;
            break;
    }
}

// filter expression
let reqresFilterDefaults = {
    sessionId: null,
    windowId: null,
    tabId: null,
    picked: null,
    was_problematic: null,
    problematic: null,
    was_in_limbo: null,
    in_limbo: null,
    collected: null,
    with_errors: null,
    did_exportAs: null,
    did_submitHTTP: null,
    in_ls: null,
    method: null,
    url_algo: null,
    url: "",
    limit: null,
};

function mkReqresFilter(value) {
    return updateFromRec(assignRec({}, reqresFilterDefaults), value);
}

function compileReqresFilter(value) {
    if (value === false) {
        return [mkReqresFilter({ limit: 0 }), (_reqres) => false];
    }
    if (value === null) {
        return [mkReqresFilter({}), (_reqres) => true];
    }

    value = mkReqresFilter(value);

    let predicates = [];

    // add predicates for the simple checks
    if (value.sessionId !== null) {
        predicates.push((reqres) => reqres.sessionId === value.sessionId);
    }
    if (value.windowId !== null) {
        predicates.push((reqres) => reqres.windowId === value.windowId);
    }
    if (value.tabId !== null) {
        predicates.push((reqres) => reqres.tabId === value.tabId);
    }
    if (value.picked !== null) {
        predicates.push((reqres) => reqres.picked === value.picked);
    }
    if (value.collected !== null) {
        predicates.push((reqres) => reqres.collected === value.collected);
    }
    if (value.problematic !== null) {
        predicates.push((reqres) => reqres.problematic === value.problematic);
    }
    if (value.was_problematic !== null) {
        predicates.push((reqres) => reqres.was_problematic === value.was_problematic);
    }
    if (value.in_limbo !== null) {
        predicates.push((reqres) => reqres.in_limbo === value.in_limbo);
    }
    if (value.was_in_limbo !== null) {
        predicates.push((reqres) => reqres.was_in_limbo === value.was_in_limbo);
    }
    if (value.with_errors !== null) {
        predicates.push((reqres) =>
            value.with_errors ? reqres.errors.length > 0 : reqres.errors.length === 0,
        );
    }
    if (value.did_exportAs !== null) {
        predicates.push((reqres) =>
            value.did_exportAs
                ? reqres.archived & (archivedViaExportAs !== 0)
                : reqres.archived & (archivedViaExportAs === 0),
        );
    }
    if (value.did_submitHTTP !== null) {
        predicates.push((reqres) =>
            value.did_submitHTTP
                ? reqres.archived & (archivedViaSubmitHTTP !== 0)
                : reqres.archived & (archivedViaSubmitHTTP === 0),
        );
    }
    if (value.in_ls !== null) {
        predicates.push((reqres) => reqres.inLS === value.in_ls);
    }
    if (value.method !== null) {
        predicates.push((reqres) => reqres.method === value.method);
    }

    // add a predicate that tests that `reqres.url` matches
    let url_algo = value.url_algo;
    let url = value.url;
    if (url_algo === false) {
        predicates.push((reqres) => reqres.url === url);
    } else if (url_algo === null) {
        if (url.length !== 0) {
            predicates.push((reqres) => reqres.url.includes(url));
        }
        // else, add nothing
    } else if (url_algo === true) {
        let re = new RegExp(url, "u");
        predicates.push((reqres) => re.test(reqres.url));
    } else {
        throw new TypeError("Bad url_algo");
    }

    return [
        value,
        (reqres) => {
            for (let predicate of predicates) {
                if (!predicate(reqres)) {
                    return false;
                }
            }
            return true;
        },
    ];
}

function escapeNotification(config, what) {
    if (config.escapeNotifications) {
        return escapeHTMLTags(what);
    }
    return what;
}

function annoyingNotification(config, what) {
    if (config.verbose) {
        return `\n\nYou can disable this notification by toggling the "${what}" option in the settings.\nYou can also toggle "User Interface and Accessibily > Verbose notifications" there to make this and similar notifications less verbose.`;
    }
    return "";
}