Support for import as module, wabac.js 2.20 (#364)

- Extensibility for AWP (customizable toolbar, pass replay iframe messages to parent)
- Support export as module, build separate index.js for module in addition to ui.js
- Update to wabac.js 2.20.0
- bump to 2.2.0

CHANGES.md

@@ -2,40 +2,53 @@
 
 ## CHANGES
 
+v2.2.0
+- Extensability: additional override options for AWP, make location toolbar more extensible
+- Extensability: support importing as module via dist/index.js, separate from ui.js
+- Deps: update to wabac.js 2.20.0, typescript support
+
 v2.1.5
+
 - Fidelity: updated rewriting rules for instagram
 - Fidelity: disable window online/offline events in replay (via wabac.js)
 - Dependencies: update to wabac.js 2.19.9 + wombat 3.8.2, electron 32.1.0
 
 v2.1.4
+
 - Fidelity: improvements to FB rewriting, eval() rewriting, 'object' tag rewriting (via wabac.js 2.19.7, wombat 3.7.14)
 
 v2.1.3
+
 - Fidelity: edge-case rewriting improvements, avoid rewriting inside of strings, detect old-html framesets (via wabac.js 2.19.5, wombat 3.7.12)
 
 v2.1.2
+
 - Fidelity: Video replay improvements, fix rewriting of DASH manifests (via wabac.js 2.19.4, wombat 3.7.11)
 
 v2.1.1
+
 - Fidelity: fixes to Sharepoint site replay (via wabac.js 2.19.2, wombat 3.7.10)
 - UI: default URL resources search to 'prefix' as contains only works after prefix search
 
 v2.1.0
+
 - More fidelity fixes (via wabac.js 2.19.1, wombat 3.7.8)
 - Improved messaging: embed tag emits @rwp-page-loading and @rwp-url-change events
 - Remove multiTs from hashtag, avoid double history nav as it was being set
 
 v2.0.2
+
 - Fidelity improvements (via wabac.js 2.18.4 and wombat 3.7.7), including improved encoding detection
 - Upgrade Shoelace & Lit by @SuaYoo in https://github.com/webrecorder/replayweb.page/pull/329
 - Adds multi-resolution ICO file for Windows by @Shrinks99 in https://github.com/webrecorder/replayweb.page/pull/325
 - Resource search remove auto-added http prefix by @ikreymer in https://github.com/webrecorder/replayweb.page/pull/334
 - Configure package exports explicitly by @emma-sg in https://github.com/webrecorder/replayweb.page/pull/336
 
 v2.0.1
+
 - Fidelity improvements (via wabac.js and wombat), including fixes ebay carousel #326 (via wombat)
 - partial fix to error page escaping #323 (via wabac)
-fixes to reddit pages #317
+- fixes to reddit pages #317
 - Fixes to twitter/x replay (via wabac / wombat)
 - Build improvements: run mkdocs gh-deploy --force in gh action, instead of publish-docs to avoid accidental updates
 - Auto install ruffle from download script, remove from git, add 'update-ruffle' script
@@ -69,7 +82,6 @@ v2.0.0
 - build(deps): bump loader-utils from 2.0.0 to 2.0.4 by @dependabot in https://github.com/webrecorder/replayweb.page/pull/316
 - 2.0.0 Release by @SuaYoo in https://github.com/webrecorder/replayweb.page/pull/269
 
-
 v1.8.17
 
 - Dependencies: update to wabac.js 2.17.3, wombat 3.7.3

README.md

@@ -72,7 +72,6 @@ The static assets are placed in the root `index.html`, `sw.js`, and `ui.js`, and
 
 The full site with docs is built using MkDocs. Using the `yarn build-docs` command, the static assets are copied into `mkdocs/site/` and using Markdown in `mkdocs/site/docs`, the final static site is built to `mkdocs/_genhtml`. This is what is published to https://replayweb.page/ via CI.
 
-
 ### Service Worker Requirements
 
 Note that for the 'backend' service worker to work, the static site must be served from either localhost or an HTTPS endpoint.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "2.1.5",
+  "version": "2.2.0",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
@@ -10,19 +10,15 @@
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",
-      "default": "./ui.js"
-    },
-    "./misc": {
-      "types": "./dist/types/misc.d.ts",
-      "default": "./dist/misc.js"
+      "default": "./dist/index.js"
     },
     "./src/electron-*": "./src/electron-*.ts",
     "./index.html": "./index.html"
   },
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.4",
     "@shoelace-style/shoelace": "~2.15.1",
-    "@webrecorder/wabac": "^2.19.9",
+    "@webrecorder/wabac": "^2.20.0",
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^6.3.2",
@@ -75,8 +71,7 @@
     "webpack-merge": "^5.10.0"
   },
   "resolutions": {
-    "**/lit": "3.1.1",
-    "@webrecorder/wabac": "^2.19.9"
+    "**/lit": "3.1.1"
   },
   "files": [
     "/dist",

scripts/build-if-not-git-repo.js

@@ -27,7 +27,7 @@ const git = (args) => cp.spawnSync("git", args, { stdio: "inherit" });
 // That's why status value needs to be checked explicitly
 if (git(["rev-parse"]).status !== 0 || process.argv.includes("--force")) {
   l(`git command not found, running build`);
-  const compiler = webpack(config.map((c) => c()));
+  const compiler = webpack(config.map((c) => c(process.env, process.argv)));
 
   compiler.run((err) => {
     if (err) {

src/appmain.ts

@@ -78,7 +78,7 @@ export class ReplayWebApp extends LitElement {
   swErrorMsg: TemplateResult<1> | string | null = null;
 
   protected swName?: string;
-  private swmanager: SWManager | null;
+  protected swmanager: SWManager | null;
   private useRuffle = false;
 
   private droppedFile: File | null = null;

src/chooser.ts

@@ -5,9 +5,15 @@ import fasUpload from "@fortawesome/fontawesome-free/svgs/solid/upload.svg";
 import { customElement, property } from "lit/decorators.js";
 
 export interface FileWithPath extends File {
-  readonly path: string;
+  path: string;
 }
 
+declare let window: Window & {
+  electron?: {
+    getPath: (file: File) => string;
+  };
+};
+
 // ===========================================================================
 @customElement("wr-chooser")
 export class Chooser extends LitElement {
@@ -89,6 +95,10 @@ export class Chooser extends LitElement {
   setFile(file: FileWithPath) {
     this.file = file;
     // file.path only available in electron app
+    if (IS_APP && window.electron?.getPath) {
+      this.file.path = window.electron.getPath(this.file);
+    }
+
     this.fileDisplayName = "file://" + (file.path || file.name);
   }
 

src/electron-preload.ts

@@ -1,12 +1,16 @@
 /*eslint-env node */
 
-// @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7016 - Could not find a declaration file for module '@webrecorder/wabac/src/loaders'. 'node_modules/@webrecorder/wabac/src/loaders.js' implicitly has an 'any' type.
-import { CollectionLoader } from "@webrecorder/wabac/src/loaders";
+import { CollectionLoader } from "@webrecorder/wabac/swlib";
 import { type IpcRendererEvent } from "electron";
 
-const { ipcRenderer, contextBridge } = require("electron");
+const { ipcRenderer, contextBridge, webUtils } = require("electron");
 
-contextBridge.exposeInMainWorld("electron", { IS_APP: true });
+contextBridge.exposeInMainWorld("electron", {
+  IS_APP: true,
+  getPath(file: File) {
+    return webUtils.getPathForFile(file);
+  },
+});
 
 const dbs = {};
 

src/electron-replay-app.ts

@@ -16,8 +16,7 @@ import {
 import path from "path";
 import fs from "fs";
 
-// @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7016 - Could not find a declaration file for module '@webrecorder/wabac/src/rewrite'. 'node_modules/@webrecorder/wabac/src/rewrite/index.js' implicitly has an 'any' type.
-import { ArchiveResponse, Rewriter } from "@webrecorder/wabac/src/rewrite";
+import { ArchiveResponse, Rewriter } from "@webrecorder/wabac";
 
 import { PassThrough, Readable } from "stream";
 
@@ -414,7 +413,7 @@ class ElectronReplayApp {
       headers = new Headers(headers);
       const date = new Date();
 
-      let response = new ArchiveResponse({
+      let response: ArchiveResponse = new ArchiveResponse({
         payload,
         headers,
         status,
@@ -443,6 +442,9 @@ class ElectronReplayApp {
         headers = Object.fromEntries(response.headers.entries());
 
         let data = await response.getBuffer();
+        if (!data) {
+          data = new Uint8Array();
+        }
 
         if (status === 206 || status === 200) {
           const { statusCode, start, end } = this.parseRange(
@@ -456,9 +458,9 @@ class ElectronReplayApp {
           status = statusCode;
         }
 
-        data = this._bufferToStream(data);
+        const result = this._bufferToStream(data);
 
-        callback({ statusCode: status, headers, data });
+        callback({ statusCode: status, headers, data: result });
       } catch (e) {
         console.warn(e);
       }

src/embed.ts

@@ -85,7 +85,7 @@ class Embed extends LitElement {
   replayfile = defaultReplayFile;
   mainElementName = "replay-app-main";
   appName = "ReplayWeb.page";
-  customConfig = null;
+  customConfig: Record<string, string | boolean> | null = null;
   reloadCount = 0;
 
   isCrossOrigin: boolean | undefined;
@@ -255,14 +255,11 @@ class Embed extends LitElement {
   }
 
   mergeConfigs() {
-    // TODO: Fix this the next time the file is edited.
-    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
     if (!this.customConfig) {
       return this.config;
     }
 
     if (this.config) {
-      // @ts-expect-error - TS2339 - Property 'customConfig' does not exist on type 'Embed'. | TS2339 - Property 'config' does not exist on type 'Embed'.
       const config = { ...this.customConfig, ...JSON.parse(this.config) };
       return JSON.stringify(config);
     } else {

src/index.ts

@@ -10,6 +10,7 @@ import { Pages } from "./pages";
 import { PageEntry } from "./pageentry";
 import { Replay } from "./replay";
 import { Sorter } from "./sorter";
+import { SWManager } from "./swmanager";
 import { URLResources } from "./url-resources";
 import { Embed } from "./embed";
 import "./shoelace";
@@ -41,6 +42,9 @@ export {
   Sorter,
   URLResources,
   Embed,
+  SWManager,
 };
 
 export type { ItemType, URLResource } from "./types";
+
+export * from "./misc";

src/item-index.ts

@@ -89,7 +89,7 @@ class ItemIndex extends LitElement {
     for (const item of this.items) {
       if (
         item.sourceUrl.indexOf(this.query) >= 0 ||
-        item.filename.indexOf(this.query) >= 0 ||
+        (item.filename && item.filename.indexOf(this.query) >= 0) ||
         Boolean(item.loadUrl && item.loadUrl.indexOf(this.query) >= 0) ||
         (item.title && item.title.indexOf(this.query) >= 0)
       ) {
@@ -383,8 +383,10 @@ class ItemIndex extends LitElement {
           >ArchiveWeb.page</a
         >
         to save pages as you browse the web, or
-        <a href="https://browsertrix.com" target="_blank">sign up for Browsertrix</a> to archive
-        entire websites with automated crawling!
+        <a href="https://browsertrix.com" target="_blank"
+          >sign up for Browsertrix</a
+        >
+        to archive entire websites with automated crawling!
       </p>
     `;
   }