software/hoardy-web/./tool/test-tool.sh

Passively capture, archive, and hoard your web browsing history, including the contents of the pages you visit, for later offline viewing, replay, mirroring, data scraping, and/or indexing. Your own personal private Wayback Machine that can also archive HTTP POST requests and responses, as well as most other HTTP-level data.

Files

Raw Source

Contents

#!/usr/bin/env bash

#set -x

. ../vendor/kisstdlib/devscript/test-cli-lib.sh

usage() {
    cat << EOF
# usage: $0 [--help] [--wine] [--all|--subset NUM] [--long|--short NUM] PATH [PATH ...]

Sanity check and test \`hoardy-web\` command-line interface.

## Examples

- Run fixed-output tests on each of given WRR bundles:

  \`\`\`
  $0 ~/Downloads/Hoardy-Web-export-*.wrrb
  \`\`\`

- Run fixed-output tests on all WRR files in a given directory:

  \`\`\`
  $0 ~/hoardy-web/latest/archiveofourown.org
  \`\`\`

- Run fixed-output tests on a random subset of WRR files in a given directory:

  \`\`\`
  $0 --subset 100 ~/hoardy-web/raw
  \`\`\`

- Run fixed-output tests on each of given WRR bundles, except run long tests on a small subset of each:

  \`\`\`
  $0 --short 16 ~/Downloads/Hoardy-Web-export-*.wrrb
  \`\`\`

- Make an \`--stdin0\` input and test on it, as if it was a WRR bundle:

  \`\`\`
  hoardy-web find -z ~/hoardy-web/latest/archiveofourown.org ~/hoardy-web/latest/example.org > ./bunch.wrrtest
  $0 ./bunch.wrrtest
  \`\`\`
EOF
}

export PYTHONPATH="$PWD:$PYTHONPATH"

in_wine=
self() {
    if [[ -z "$in_wine" ]]; then
        python3 -m hoardy_web "$@"
    else
        wine python -m hoardy_web "$@"
    fi
}

ok_selfsame() {
    local got="$1"
    local input="$2"
    local stdin0="$3"
    shift 3

    ok_stdio2 "$got.out" "$@" "$input"
    ok_stdio2 "$got.stdin0.out" "$@" --stdin0 < "$stdin0"
    equal_file "$got.out" "$got.stdin0.out"
}

fixed_stdio() {
    local src="$1"
    local got="$2"
    shift 2

    ok_stdio2 "$got.out" "$@"
    sed -i "s%$tmpdir/%./%g" "$got.out"
    fixed_file "$src" "$got.out"
}

fixed_stdio_selfsame() {
    local src="$1"
    local got="$2"
    local input="$3"
    local stdin0="$4"
    shift 4

    ok_selfsame "$got" "$input" "$stdin0" "$@"
    sed -i "s%$tmpdir/%./%g" "$got.out"
    fixed_file "$src" "$got.out"
}

[[ $# < 1 ]] && die "need at least one source"

opts=1
subset=
short=

while (($# > 0)); do
    if [[ -n "$opts" ]]; then
        case "$1" in
        --help) usage; exit 0; ;;
        --wine) in_wine=1 ; shift ; continue ;;
        --all) subset= ; shift ; continue ;;
        --subset) subset=$2 ; shift 2 ; continue ;;
        --long) short= ; shift ; continue ;;
        --short) short=$2 ; shift 2 ; continue ;;
        --) opts= ; shift ; continue ;;
        esac
    fi

    src=$1
    shift

    set_tmpdir

    do_fixed_dir=1
    if [[ -z "$in_wine" ]]; then
        if [[ -f "$src" ]] && [[ "$src" =~ .*\.wrrb ]]; then
            # these can be made with `cat`ting a bunch of .wrr files together
            echo "# Testing fixed-outputness on bundle $src in $tmpdir ..."
            stdin0="$tmpdir/input"
            find "$src" -type f -print0 > "$stdin0"
        elif [[ -f "$src" ]]; then
            # these can be made with `hoardy-web find -z`
            echo "# Testing fixed-outputness on stdin0 $src in $tmpdir ..."
            stdin0="$src"
        elif [[ -d "$src" ]]; then
            stdin0="$tmpdir/input"
            if [[ -z "$subset" ]]; then
                echo "# Testing fixed-outputness on whole dir $src in $tmpdir ..."
                find "$src" -type f -print0 | sort -z > "$stdin0"
            else
                echo "# Testing fixed-outputness on a random subset (n=$subset) of dir $src in $tmpdir ..."
                find "$src" -type f -print0 | shuf -z | head -zn "$subset" > "$stdin0"
                do_fixed_dir=
            fi
        else
            error "can't run tests on $src"
            continue
        fi
    else
        if [[ -f "$src" ]] && [[ "$src" =~ .*\.wrrb ]]; then
            echo "# Testing fixed-outputness on bundle $src in $tmpdir ..."
        elif [[ -f "$src" ]]; then
            error "testing on $src is not supported"
            continue
        elif [[ -d "$src" ]]; then
            echo "# Testing fixed-outputness on whole dir $src in $tmpdir ..."
            find "$src" -type f -print0 | win32path0_many > "$tmpdir/src"
            src="$tmpdir/src"
        else
            error "can't run tests on $src"
            continue
        fi
    fi

    uexprs=( \
        -e net_url \
        -e url \
        -e pretty_url \
    )
    exprs=( \
        "${uexprs[@]}" \
        -e "net_url|to_ascii|sha256|take_prefix 2|to_hex" \
        -e "response.body|eb|to_utf8|sha256|to_hex" \
    )

    cd "$tmpdir"

    start "import bundle"

    if [[ -z "$in_wine" ]]; then
        ok_no_stderr self import bundle --quiet --stdin0 --to import-bundle < "$stdin0"
    else
        ok self import bundle --quiet --to import-bundle "$(win32path "$src")"
    fi
    [[ -n "$do_fixed_dir" ]] && fixed_dir "$src" import-bundle

    end

    idir=import-bundle

    input0=input0
    find "$idir" -type f -print0 | sort -z > "$input0"

    if [[ -n "$short" ]]; then
        sinput0=sinput0
        cat "$input0" | shuf -z | head -zn "$short" > "$sinput0"
    else
        sinput0="$input0"
    fi

    if [[ -z "$in_wine" ]]; then
        start "filter out \`.part\`s"

        ok_stdio2 dotpart.1.out self stream --format=raw -ue url "$idir"

        while IFS= read -r -d $'\0' fname; do
            cp "$fname" "$fname.part"
        done < "$sinput0"

        ok_stdio2 dotpart.2.out self stream --errors skip --format=raw -ue url "$idir"

        while IFS= read -r -d $'\0' fname; do
            rm "$fname.part"
        done < "$sinput0"

        equal_file dotpart.1.out dotpart.2.out

        end

        start find

        fixed_stdio_selfsame "$src" find-200-1024 "$idir" "$input0" \
                             self find --status-re .200C --and "response.body|len|> 1024"

        fixed_stdio_selfsame "$src" find-html-potter "$idir" "$input0" \
                             self find --response-mime text/html --grep-re '\bPotter\b'

        end

        start pprint

        ok_selfsame pprint "$idir" "$input0" self pprint
        ok_selfsame pprint-u "$idir" "$input0" self pprint -u

        end

        start stream

        ok_selfsame stream "$idir" "$input0" self stream "${exprs[@]}"
        ok_selfsame stream-u "$idir" "$input0" self stream -u "${exprs[@]}"

        end

        start organize

        ok_no_stderr self organize --quiet --copy --to organize "$idir"
        equal_dir organize "$idir"

        ok_no_stderr self organize --quiet --hardlink --to organize2 organize
        equal_dir organize2 "$idir"

        ok_no_stderr self organize --quiet --symlink --output hupq_msn \
                       --to organize3 organize

        {
            ok self organize --copy --to organize organize2
            ok self organize --hardlink --to organize organize2
            ok self organize --copy --to organize2 organize
            ok self organize --hardlink --to organize2 organize

            ok self organize --hardlink --to organize organize3
            ok self organize --symlink --output hupq_msn --to organize3 organize
        } &> reorganize.log

        if [[ -s reorganize.log ]]; then
            cat reorganize.log
            die "re-organize is not a noop"
        fi

        end

        start "organize --symlink --latest"

        fixed_stdio "$src" organize-sl \
                    self organize --symlink --latest --output hupq \
                    --to organize-sl \
                    "$idir"

        fixed_stdio "$src" organize-sls \
                    self organize --symlink --latest --output hupq \
                    --paths-sorted --walk-sorted \
                    --to organize-sls \
                    "$idir"

        # TODO: this, currently broken
        # equal_dir organize-sl organize-sls

        lines=$(cat "$input0" | tr '\0' '\n' | wc -l)

        cat "$input0" | head -zn $((lines/3 + 1)) | \
            fixed_stdio "$src" organize-seq1 \
                        self organize --symlink --latest --output hupq \
                        --to organize-seq --stdin0

        cat "$input0" | head -zn $((lines*2/3 + 1)) | \
            fixed_stdio "$src" organize-seq2 \
                        self organize --symlink --latest --output hupq \
                        --to organize-seq --stdin0

        cat "$input0" | tail -zn $((lines*2/3 + 1)) | \
            fixed_stdio "$src" organize-seq3 \
                        self organize --symlink --latest --output hupq \
                        --to organize-seq --stdin0

        equal_dir organize-sl organize-seq

        # ensure `organize` did not touch the source dir
        describe-forest import-bundle > import-bundle.describe-dir.2
        equal_file import-bundle.describe-dir import-bundle.describe-dir.2

        end
    fi

    start "serve archival"
    # feed results of `import bundle` to `serve` via `curl`, then check
    # that the results are the same

    mkdir -p serve
    if [[ -z "$in_wine" ]]; then
        python3 -m hoardy_web serve --host 127.1.1.1 --implicit --archive-to serve &
    else
        wine python -m hoardy_web serve --host 127.1.1.1 --implicit --archive-to serve &
    fi
    tmppid=$!
    sleep 3

    # just to be sure
    curl "http://127.1.1.1:3210/hoardy-web/server-info" > serve.info 2> /dev/null
    fixed_file "$src" serve.info

    # feed it some data
    while IFS= read -r -d $'\0' fname; do
        zcat "$fname" | curl --data-binary "@-" -H "Content-type: application/x-wrr+cbor" "http://127.1.1.1:3210/pwebarc/dump"
    done < "$sinput0"

    # kill immediately, which must work
    kill "$tmppid"
    tmppid=

    # ensure no .part files are left
    find serve -name '*.part' > serve.parts

    if [[ -s serve.parts ]]; then
        cat serve.parts
        error "serve left some \`.part\` files"
    fi

    # check equality to import-bundle
    while IFS= read -r -d $'\0' fname; do
        if ! diff "$fname" "serve/default/${fname#import-bundle/}" > /dev/null ; then
            error "$fname is not the same"
        fi
    done < "$sinput0"

    end

    if [[ -z "$in_wine" ]]; then
        start "mirror urls"

        fixed_stdio "$src" mirror-urls \
                    self mirror --copy --to mirror-urls --output hupq_n \
                    "${uexprs[@]}" \
                    "$idir"
        [[ -n "$do_fixed_dir" ]] && fixed_dir "$src" mirror-urls

        end

        start "mirror responses"

        fixed_stdio "$src" mirror-responses \
                    self mirror --to mirror-responses --output hupq_n \
                    "$idir"
        [[ -n "$do_fixed_dir" ]] && fixed_dir "$src" mirror-responses

        end

        start get

        while IFS= read -r -d $'\0' path; do
            ok_no_stderr self get "${exprs[@]}" "$path"
            ok_no_stderr self get --sniff-force "${exprs[@]}" "$path"
            ok_no_stderr self get --sniff-paranoid "${exprs[@]}" "$path"
        done < "$sinput0"

        end

        start run

        while IFS= read -r -d $'\0' path; do
            ok_no_stderr self run cat "$path"
            ok_no_stderr self run -n 2 -- diff "$path" "$path"
        done < "$sinput0"

        end

        start "stream --format=raw"

        ok_selfsame stream-raw   "$idir" "$input0" self stream --format=raw "${exprs[@]}"
        ok_selfsame stream-raw-u "$idir" "$input0" self stream --format=raw -u "${exprs[@]}"

        end

        start "stream --format=json"

        ok_selfsame stream-json    "$idir" "$input0" self stream --format=json "${exprs[@]}"
        ok_selfsame stream-json-u  "$idir" "$input0" self stream --format=json -u "${exprs[@]}"

        end

        #start "stream --format=cbor"

        #ok_selfsame stream-cbor    "$idir" "$input0" self stream --format=cbor "${exprs[@]}"
        #ok_selfsame stream-cbor-u  "$idir" "$input0" self stream --format=cbor -u "${exprs[@]}"

        #end
    fi

    cd /
    rm -rf "$tmpdir"
done

finish