Replace to `workbox-webpack-plugin` from `offline-plugin` (#18409)

app/javascript/mastodon/main.js

@@ -1,9 +1,9 @@
-import * as registerPushNotifications from './actions/push_notifications';
-import { setupBrowserNotifications } from './actions/notifications';
-import { default as Mastodon, store } from './containers/mastodon';
 import React from 'react';
 import ReactDOM from 'react-dom';
-import ready from './ready';
+import * as registerPushNotifications from 'mastodon/actions/push_notifications';
+import { setupBrowserNotifications } from 'mastodon/actions/notifications';
+import Mastodon, { store } from 'mastodon/containers/mastodon';
+import ready from 'mastodon/ready';
 
 const perf = require('./performance');
 
@@ -24,10 +24,20 @@ function main() {
 
     ReactDOM.render(<Mastodon {...props} />, mountNode);
     store.dispatch(setupBrowserNotifications());
-    if (process.env.NODE_ENV === 'production') {
-      // avoid offline in dev mode because it's harder to debug
-      require('offline-plugin/runtime').install();
-      store.dispatch(registerPushNotifications.register());
+
+    if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
+      import('workbox-window')
+        .then(({ Workbox }) => {
+          const wb = new Workbox('/sw.js');
+
+          return wb.register();
+        })
+        .then(() => {
+          store.dispatch(registerPushNotifications.register());
+        })
+        .catch(err => {
+          console.error(err);
+        });
     }
     perf.stop('main()');
   });

app/javascript/mastodon/service_worker/entry.js

@@ -1,20 +1,59 @@
-// import { freeStorage, storageFreeable } from '../storage/modifier';
-import './web_push_notifications';
+import { ExpirationPlugin } from 'workbox-expiration';
+import { precacheAndRoute } from 'workbox-precaching';
+import { registerRoute } from 'workbox-routing';
+import { CacheFirst } from 'workbox-strategies';
+import { handleNotificationClick, handlePush } from './web_push_notifications';
 
-// function openSystemCache() {
-//   return caches.open('mastodon-system');
-// }
+const CACHE_NAME_PREFIX = 'mastodon-';
 
 function openWebCache() {
-  return caches.open('mastodon-web');
+  return caches.open(`${CACHE_NAME_PREFIX}web`);
 }
 
 function fetchRoot() {
   return fetch('/', { credentials: 'include', redirect: 'manual' });
 }
 
-// const firefox = navigator.userAgent.match(/Firefox\/(\d+)/);
-// const invalidOnlyIfCached = firefox && firefox[1] < 60;
+precacheAndRoute(self.__WB_MANIFEST);
+
+registerRoute(
+  /locale_.*\.js$/,
+  new CacheFirst({
+    cacheName: `${CACHE_NAME_PREFIX}locales`,
+    plugins: [
+      new ExpirationPlugin({
+        maxAgeSeconds: 30 * 24 * 60 * 60, // 1 month
+        maxEntries: 5,
+      }),
+    ],
+  }),
+);
+
+registerRoute(
+  ({ request }) => request.destination === 'font',
+  new CacheFirst({
+    cacheName: `${CACHE_NAME_PREFIX}fonts`,
+    plugins: [
+      new ExpirationPlugin({
+        maxAgeSeconds: 30 * 24 * 60 * 60, // 1 month
+        maxEntries: 5,
+      }),
+    ],
+  }),
+);
+
+registerRoute(
+  ({ request }) => ['audio', 'image', 'track', 'video'].includes(request.destination),
+  new CacheFirst({
+    cacheName: `m${CACHE_NAME_PREFIX}media`,
+    plugins: [
+      new ExpirationPlugin({
+        maxAgeSeconds: 7 * 24 * 60 * 60, // 1 week
+        maxEntries: 256,
+      }),
+    ],
+  }),
+);
 
 // Cause a new version of a registered Service Worker to replace an existing one
 // that is already installed, and replace the currently active worker on open pages.
@@ -52,26 +91,8 @@ self.addEventListener('fetch', function(event) {
 
       return response;
     }));
-  } /* else if (storageFreeable && (ATTACHMENT_HOST ? url.host === ATTACHMENT_HOST : url.pathname.startsWith('/system/'))) {
-    event.respondWith(openSystemCache().then(cache => {
-      return cache.match(event.request.url).then(cached => {
-        if (cached === undefined) {
-          const asyncResponse = invalidOnlyIfCached && event.request.cache === 'only-if-cached' ?
-            fetch(event.request, { cache: 'no-cache' }) : fetch(event.request);
-
-          return asyncResponse.then(response => {
-            if (response.ok) {
-              cache
-                .put(event.request.url, response.clone())
-                .catch(()=>{}).then(freeStorage()).catch();
-            }
-
-            return response;
-          });
-        }
-
-        return cached;
-      });
-    }));
-  } */
+  }
 });
+
+self.addEventListener('push', handlePush);
+self.addEventListener('notificationclick', handleNotificationClick);

app/javascript/mastodon/service_worker/web_push_notifications.js

@@ -75,7 +75,7 @@ const formatMessage = (messageId, locale, values = {}) =>
 const htmlToPlainText = html =>
   unescape(html.replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n').replace(/<[^>]*>/g, ''));
 
-const handlePush = (event) => {
+export const handlePush = (event) => {
   const { access_token, notification_id, preferred_locale, title, body, icon } = event.data.json();
 
   // Placeholder until more information can be loaded
@@ -189,7 +189,7 @@ const openUrl = url =>
     return self.clients.openWindow(url);
   });
 
-const handleNotificationClick = (event) => {
+export const handleNotificationClick = (event) => {
   const reactToNotificationClick = new Promise((resolve, reject) => {
     if (event.action) {
       if (event.action === 'expand') {
@@ -211,6 +211,3 @@ const handleNotificationClick = (event) => {
 
   event.waitUntil(reactToNotificationClick);
 };
-
-self.addEventListener('push', handlePush);
-self.addEventListener('notificationclick', handleNotificationClick);

app/javascript/mastodon/storage/db.js

@@ -1,27 +0,0 @@
-export default () => new Promise((resolve, reject) => {
-  // ServiceWorker is required to synchronize the login state.
-  // Microsoft Edge 17 does not support getAll according to:
-  // Catalog of standard and vendor APIs across browsers - Microsoft Edge Development
-  // https://developer.microsoft.com/en-us/microsoft-edge/platform/catalog/?q=specName%3Aindexeddb
-  if (!('caches' in self && 'getAll' in IDBObjectStore.prototype)) {
-    reject();
-    return;
-  }
-
-  const request = indexedDB.open('mastodon');
-
-  request.onerror = reject;
-  request.onsuccess = ({ target }) => resolve(target.result);
-
-  request.onupgradeneeded = ({ target }) => {
-    const accounts = target.result.createObjectStore('accounts', { autoIncrement: true });
-    const statuses = target.result.createObjectStore('statuses', { autoIncrement: true });
-
-    accounts.createIndex('id', 'id', { unique: true });
-    accounts.createIndex('moved', 'moved');
-
-    statuses.createIndex('id', 'id', { unique: true });
-    statuses.createIndex('account', 'account');
-    statuses.createIndex('reblog', 'reblog');
-  };
-});

app/javascript/mastodon/storage/modifier.js

@@ -1,211 +0,0 @@
-import openDB from './db';
-
-const accountAssetKeys = ['avatar', 'avatar_static', 'header', 'header_static'];
-const storageMargin = 8388608;
-const storeLimit = 1024;
-
-// navigator.storage is not present on:
-// Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.100 Safari/537.36 Edge/16.16299
-// estimate method is not present on Chrome 57.0.2987.98 on Linux.
-export const storageFreeable = 'storage' in navigator && 'estimate' in navigator.storage;
-
-function openCache() {
-  // ServiceWorker and Cache API is not available on iOS 11
-  // https://webkit.org/status/#specification-service-workers
-  return self.caches ? caches.open('mastodon-system') : Promise.reject();
-}
-
-function printErrorIfAvailable(error) {
-  if (error) {
-    console.warn(error);
-  }
-}
-
-function put(name, objects, onupdate, oncreate) {
-  return openDB().then(db => (new Promise((resolve, reject) => {
-    const putTransaction = db.transaction(name, 'readwrite');
-    const putStore = putTransaction.objectStore(name);
-    const putIndex = putStore.index('id');
-
-    objects.forEach(object => {
-      putIndex.getKey(object.id).onsuccess = retrieval => {
-        function addObject() {
-          putStore.add(object);
-        }
-
-        function deleteObject() {
-          putStore.delete(retrieval.target.result).onsuccess = addObject;
-        }
-
-        if (retrieval.target.result) {
-          if (onupdate) {
-            onupdate(object, retrieval.target.result, putStore, deleteObject);
-          } else {
-            deleteObject();
-          }
-        } else {
-          if (oncreate) {
-            oncreate(object, addObject);
-          } else {
-            addObject();
-          }
-        }
-      };
-    });
-
-    putTransaction.oncomplete = () => {
-      const readTransaction = db.transaction(name, 'readonly');
-      const readStore = readTransaction.objectStore(name);
-      const count = readStore.count();
-
-      count.onsuccess = () => {
-        const excess = count.result - storeLimit;
-
-        if (excess > 0) {
-          const retrieval = readStore.getAll(null, excess);
-
-          retrieval.onsuccess = () => resolve(retrieval.result);
-          retrieval.onerror = reject;
-        } else {
-          resolve([]);
-        }
-      };
-
-      count.onerror = reject;
-    };
-
-    putTransaction.onerror = reject;
-  })).then(resolved => {
-    db.close();
-    return resolved;
-  }, error => {
-    db.close();
-    throw error;
-  }));
-}
-
-function evictAccountsByRecords(records) {
-  return openDB().then(db => {
-    const transaction = db.transaction(['accounts', 'statuses'], 'readwrite');
-    const accounts = transaction.objectStore('accounts');
-    const accountsIdIndex = accounts.index('id');
-    const accountsMovedIndex = accounts.index('moved');
-    const statuses = transaction.objectStore('statuses');
-    const statusesIndex = statuses.index('account');
-
-    function evict(toEvict) {
-      toEvict.forEach(record => {
-        openCache()
-          .then(cache => accountAssetKeys.forEach(key => cache.delete(records[key])))
-          .catch(printErrorIfAvailable);
-
-        accountsMovedIndex.getAll(record.id).onsuccess = ({ target }) => evict(target.result);
-
-        statusesIndex.getAll(record.id).onsuccess =
-          ({ target }) => evictStatusesByRecords(target.result);
-
-        accountsIdIndex.getKey(record.id).onsuccess =
-          ({ target }) => target.result && accounts.delete(target.result);
-      });
-    }
-
-    evict(records);
-
-    db.close();
-  }).catch(printErrorIfAvailable);
-}
-
-export function evictStatus(id) {
-  evictStatuses([id]);
-}
-
-export function evictStatuses(ids) {
-  return openDB().then(db => {
-    const transaction = db.transaction('statuses', 'readwrite');
-    const store = transaction.objectStore('statuses');
-    const idIndex = store.index('id');
-    const reblogIndex = store.index('reblog');
-
-    ids.forEach(id => {
-      reblogIndex.getAllKeys(id).onsuccess =
-        ({ target }) => target.result.forEach(reblogKey => store.delete(reblogKey));
-
-      idIndex.getKey(id).onsuccess =
-        ({ target }) => target.result && store.delete(target.result);
-    });
-
-    db.close();
-  }).catch(printErrorIfAvailable);
-}
-
-function evictStatusesByRecords(records) {
-  return evictStatuses(records.map(({ id }) => id));
-}
-
-export function putAccounts(records, avatarStatic) {
-  const avatarKey = avatarStatic ? 'avatar_static' : 'avatar';
-  const newURLs = [];
-
-  put('accounts', records, (newRecord, oldKey, store, oncomplete) => {
-    store.get(oldKey).onsuccess = ({ target }) => {
-      accountAssetKeys.forEach(key => {
-        const newURL = newRecord[key];
-        const oldURL = target.result[key];
-
-        if (newURL !== oldURL) {
-          openCache()
-            .then(cache => cache.delete(oldURL))
-            .catch(printErrorIfAvailable);
-        }
-      });
-
-      const newURL = newRecord[avatarKey];
-      const oldURL = target.result[avatarKey];
-
-      if (newURL !== oldURL) {
-        newURLs.push(newURL);
-      }
-
-      oncomplete();
-    };
-  }, (newRecord, oncomplete) => {
-    newURLs.push(newRecord[avatarKey]);
-    oncomplete();
-  }).then(records => Promise.all([
-    evictAccountsByRecords(records),
-    openCache().then(cache => cache.addAll(newURLs)),
-  ])).then(freeStorage, error => {
-    freeStorage();
-    throw error;
-  }).catch(printErrorIfAvailable);
-}
-
-export function putStatuses(records) {
-  put('statuses', records)
-    .then(evictStatusesByRecords)
-    .catch(printErrorIfAvailable);
-}
-
-export function freeStorage() {
-  return storageFreeable && navigator.storage.estimate().then(({ quota, usage }) => {
-    if (usage + storageMargin < quota) {
-      return null;
-    }
-
-    return openDB().then(db => new Promise((resolve, reject) => {
-      const retrieval = db.transaction('accounts', 'readonly').objectStore('accounts').getAll(null, 1);
-
-      retrieval.onsuccess = () => {
-        if (retrieval.result.length > 0) {
-          resolve(evictAccountsByRecords(retrieval.result).then(freeStorage));
-        } else {
-          resolve(caches.delete('mastodon-system'));
-        }
-      };
-
-      retrieval.onerror = reject;
-
-      db.close();
-    }));
-  });
-}

config/webpack/production.js

@@ -1,29 +1,16 @@
 // Note: You must restart bin/webpack-dev-server for changes to take effect
 
-const path = require('path');
-const { URL } = require('url');
+const { createHash } = require('crypto');
+const { readFileSync } = require('fs');
+const { resolve } = require('path');
 const { merge } = require('webpack-merge');
 const { BundleAnalyzerPlugin } = require('webpack-bundle-analyzer');
-const OfflinePlugin = require('offline-plugin');
 const TerserPlugin = require('terser-webpack-plugin');
 const CompressionPlugin = require('compression-webpack-plugin');
-const { output } = require('./configuration');
+const { InjectManifest } = require('workbox-webpack-plugin');
 const sharedConfig = require('./shared');
 
-let attachmentHost;
-
-if (process.env.S3_ENABLED === 'true') {
-  if (process.env.S3_ALIAS_HOST || process.env.S3_CLOUDFRONT_HOST) {
-    attachmentHost = process.env.S3_ALIAS_HOST || process.env.S3_CLOUDFRONT_HOST;
-  } else {
-    attachmentHost = process.env.S3_HOSTNAME || `s3-${process.env.S3_REGION || 'us-east-1'}.amazonaws.com`;
-  }
-} else if (process.env.SWIFT_ENABLED === 'true') {
-  const { host } = new URL(process.env.SWIFT_OBJECT_URL);
-  attachmentHost = host;
-} else {
-  attachmentHost = null;
-}
+const root = resolve(__dirname, '..', '..');
 
 module.exports = merge(sharedConfig, {
   mode: 'production',
@@ -52,47 +39,28 @@ module.exports = merge(sharedConfig, {
       openAnalyzer: false,
       logLevel: 'silent', // do not bother Webpacker, who runs with --json and parses stdout
     }),
-    new OfflinePlugin({
-      publicPath: output.publicPath, // sw.js must be served from the root to avoid scope issues
-      safeToUseOptionalCaches: true,
-      caches: {
-        main: [':rest:'],
-        additional: [':externals:'],
-        optional: [
-          '**/locale_*.js', // don't fetch every locale; the user only needs one
-          '**/*_polyfills-*.js', // the user may not need polyfills
-          '**/*.woff2', // the user may have system-fonts enabled
-          // images/audio can be cached on-demand
-          '**/*.png',
-          '**/*.jpg',
-          '**/*.jpeg',
-          '**/*.svg',
-          '**/*.mp3',
-          '**/*.ogg',
-        ],
-      },
-      externals: [
-        '/emoji/1f602.svg', // used for emoji picker dropdown
-        '/emoji/sheet_10.png', // used in emoji-mart
-      ],
-      excludes: [
-        '**/*.gz',
-        '**/*.map',
-        'stats.json',
-        'report.html',
-        // any browser that supports ServiceWorker will support woff2
-        '**/*.eot',
-        '**/*.ttf',
-        '**/*-webfont-*.svg',
-        '**/*.woff',
+    new InjectManifest({
+      additionalManifestEntries: ['1f602.svg', 'sheet_13.png'].map((filename) => {
+        const path = resolve(root, 'public', 'emoji', filename);
+        const body = readFileSync(path);
+        const md5  = createHash('md5');
+
+        md5.update(body);
+
+        return {
+          revision: md5.digest('hex'),
+          url: `/emoji/${filename}`,
+        };
+      }),
+      exclude: [
+        /(?:base|extra)_polyfills-.*\.js$/,
+        /locale_.*\.js$/,
+        /mailer-.*\.(?:css|js)$/,
       ],
-      ServiceWorker: {
-        entry: `imports-loader?additionalCode=${encodeURIComponent(`var ATTACHMENT_HOST=${JSON.stringify(attachmentHost)};`)}!${encodeURI(path.join(__dirname, '../../app/javascript/mastodon/service_worker/entry.js'))}`,
-        cacheName: 'mastodon',
-        output: '../assets/sw.js',
-        publicPath: '/sw.js',
-        minify: true,
-      },
+      include: [/\.js$/, /\.css$/],
+      maximumFileSizeToCacheInBytes: 2 * 1_024 * 1_024, // 2 MiB
+      swDest: resolve(root, 'public', 'packs', 'sw.js'),
+      swSrc: resolve(root, 'app', 'javascript', 'mastodon', 'service_worker', 'entry.js'),
     }),
   ],
 });

package.json

@@ -82,7 +82,6 @@
     "object-assign": "^4.1.1",
     "object-fit-images": "^3.2.3",
     "object.values": "^1.1.5",
-    "offline-plugin": "^5.0.7",
     "path-complete-extname": "^1.0.0",
     "pg": "^8.5.0",
     "postcss": "^8.4.16",
@@ -136,6 +135,12 @@
     "webpack-cli": "^3.3.12",
     "webpack-merge": "^5.8.0",
     "wicg-inert": "^3.1.2",
+    "workbox-expiration": "^6.5.3",
+    "workbox-precaching": "^6.5.3",
+    "workbox-routing": "^6.5.3",
+    "workbox-strategies": "^6.5.3",
+    "workbox-webpack-plugin": "^6.5.3",
+    "workbox-window": "^6.5.3",
     "ws": "^8.8.1"
   },
   "devDependencies": {

public/sw.js

@@ -1 +1 @@
-assets/sw.js
+packs/sw.js

public/sw.js.map

@@ -0,0 +1 @@
+packs/sw.js.map