fileStorage: new file storage backend

This replaces the localStorage file storage with a new backend that uses
localForage (indexeddb) for storing the file contents. This will also
allow storing multiple files.

src/fileStorage/actions.ts

@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 The Pybricks Authors
+
+import { Action } from 'redux';
+
+export enum FileStorageActionType {
+    /** Action that indicates that the storage backend is ready to use. */
+    DidInitialize = 'fileStorage.action.didInitialize',
+    /** Action that indicates that the storage backend failed to initialize. */
+    DidFailToInitialize = 'fileStorage.action.didFailToInitialize',
+    /** Action that indicates that an item in the storage was created or changed by us or in another tab. */
+    DidChangeItem = 'fileStorage.action.didChangeItem',
+    /** Action that indicates that an item in the storage was removed by us or in another tab. */
+    DidRemoveItem = 'fileStorage.action.didRemoveItem',
+    /** Requests to read a file from storage. */
+    ReadFile = 'fileStorage.action.readFile',
+    /** Response to read file request indicating success. */
+    DidReadFile = 'fileStorage.action.didReadFile',
+    /** Response to read file request indicating failure. */
+    DidFailToReadFile = 'fileStorage.action.didFailToReadFile',
+    /** Requests to write a file to storage. */
+    WriteFile = 'fileStorage.action.writeFile',
+    /** Response to write file request indicating success. */
+    DidWriteFile = 'fileStorage.action.didWriteFile',
+    /** Response to write file request indicating failure. */
+    DidFailToWriteFile = 'fileStorage.action.didFailToWriteFile',
+}
+
+/** Action that indicates that the storage backend is ready to use. */
+export type FileStorageDidInitializeAction =
+    Action<FileStorageActionType.DidInitialize>;
+
+/** Action that indicates that the storage backend is ready to use. */
+export function fileStorageDidInitialize(): FileStorageDidInitializeAction {
+    return { type: FileStorageActionType.DidInitialize };
+}
+
+/** Action that indicates that the storage backend failed to initialize. */
+export type FileStorageDidFailToInitializeAction =
+    Action<FileStorageActionType.DidFailToInitialize> & { error: Error };
+
+/** Action that indicates that the storage backend failed to initialize. */
+export function fileStorageDidFailToInitialize(
+    err: Error,
+): FileStorageDidFailToInitializeAction {
+    return { type: FileStorageActionType.DidFailToInitialize, error: err };
+}
+
+/** Action that indicates that an item in the storage was created or changed by us or in another tab. */
+export type FileStorageDidChangeItemAction =
+    Action<FileStorageActionType.DidChangeItem> & {
+        fileName: string;
+    };
+
+/** Action that indicates that an item in the storage was created or changed by us or in another tab. */
+export function fileStorageDidChangeItem(
+    fileName: string,
+): FileStorageDidChangeItemAction {
+    return { type: FileStorageActionType.DidChangeItem, fileName };
+}
+
+/** Action that indicates that an item in the storage was removed by us or in another tab. */
+export type FileStorageDidRemoveItemAction =
+    Action<FileStorageActionType.DidRemoveItem> & {
+        fileName: string;
+    };
+
+/** Action that indicates that an item in the storage was removed by us or in another tab. */
+export function fileStorageDidRemoveItem(
+    fileName: string,
+): FileStorageDidRemoveItemAction {
+    return { type: FileStorageActionType.DidRemoveItem, fileName };
+}
+
+/** Requests to read a file from storage. */
+export type FileStorageReadFileAction = Action<FileStorageActionType.ReadFile> & {
+    fileName: string;
+};
+
+/** Requests to read a file from storage. */
+export function fileStorageReadFile(fileName: string): FileStorageReadFileAction {
+    return { type: FileStorageActionType.ReadFile, fileName };
+}
+
+/** Response to read file request indicating success. */
+export type FileStorageDidReadFileAction = Action<FileStorageActionType.DidReadFile> & {
+    fileName: string;
+    fileContents: string;
+};
+
+/** Response to read file request indicating success. */
+export function fileStorageDidReadFile(
+    fileName: string,
+    fileContents: string,
+): FileStorageDidReadFileAction {
+    return { type: FileStorageActionType.DidReadFile, fileName, fileContents };
+}
+
+/** Response to read file request indicating failure. */
+export type FileStorageDidFailToReadFileAction =
+    Action<FileStorageActionType.DidFailToReadFile> & {
+        fileName: string;
+        error: Error;
+    };
+
+/** Response to read file request indicating failure. */
+export function fileStorageDidFailToReadFile(
+    fileName: string,
+    error: Error,
+): FileStorageDidFailToReadFileAction {
+    return { type: FileStorageActionType.DidFailToReadFile, fileName, error };
+}
+
+/** Requests to write a file to storage. */
+export type FileStorageWriteFileAction = Action<FileStorageActionType.WriteFile> & {
+    fileName: string;
+    fileContents: string;
+};
+
+/** Requests to write a file to storage. */
+export function fileStorageWriteFile(
+    fileName: string,
+    fileContents: string,
+): FileStorageWriteFileAction {
+    return { type: FileStorageActionType.WriteFile, fileName, fileContents };
+}
+
+/** Response to write file request indicating success. */
+export type FileStorageDidWriteFileAction =
+    Action<FileStorageActionType.DidWriteFile> & {
+        fileName: string;
+    };
+
+/** Response to write file request indicating success. */
+export function fileStorageDidWriteFile(
+    fileName: string,
+): FileStorageDidWriteFileAction {
+    return { type: FileStorageActionType.DidWriteFile, fileName };
+}
+
+/** Response to write file request indicating failure. */
+export type FileStorageDidFailToWriteFileAction =
+    Action<FileStorageActionType.DidFailToWriteFile> & {
+        fileName: string;
+        error: Error;
+    };
+
+/** Response to write file request indicating failure. */
+export function fileStorageDidFailToWriteFile(
+    fileName: string,
+    error: Error,
+): FileStorageDidFailToWriteFileAction {
+    return { type: FileStorageActionType.DidFailToWriteFile, fileName, error };
+}
+
+/**
+ * Common type for all file storage actions.
+ */
+export type FileStorageAction =
+    | FileStorageDidInitializeAction
+    | FileStorageDidFailToInitializeAction
+    | FileStorageDidChangeItemAction
+    | FileStorageDidRemoveItemAction
+    | FileStorageReadFileAction
+    | FileStorageDidReadFileAction
+    | FileStorageDidFailToReadFileAction
+    | FileStorageWriteFileAction
+    | FileStorageDidWriteFileAction
+    | FileStorageDidFailToWriteFileAction;

src/fileStorage/reducers.test.ts

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 The Pybricks Authors
+
+import { Action } from '../actions';
+import {
+    fileStorageDidChangeItem,
+    fileStorageDidInitialize,
+    fileStorageDidRemoveItem,
+} from './actions';
+import reducers from './reducers';
+
+type State = ReturnType<typeof reducers>;
+
+test('initial state', () => {
+    expect(reducers(undefined, {} as Action)).toMatchInlineSnapshot(`
+        Object {
+          "fileNames": Set {},
+          "isInitialized": false,
+        }
+    `);
+});
+
+test('isInitialized', () => {
+    expect(
+        reducers({ isInitialized: false } as State, fileStorageDidInitialize())
+            .isInitialized,
+    ).toBeTruthy();
+});
+
+test('fileNames', () => {
+    const testFileName = 'test.file';
+
+    // if item is not in set, add it
+    expect(
+        reducers(
+            { fileNames: new Set() } as State,
+            fileStorageDidChangeItem(testFileName),
+        ).fileNames,
+    ).toEqual(new Set([testFileName]));
+
+    // if item is already in set, there should not be duplicates
+    expect(
+        reducers(
+            { fileNames: new Set([testFileName]) } as State,
+            fileStorageDidChangeItem(testFileName),
+        ).fileNames,
+    ).toEqual(new Set([testFileName]));
+
+    // if item is in set, it should be removed
+    expect(
+        reducers(
+            { fileNames: new Set([testFileName]) } as State,
+            fileStorageDidRemoveItem(testFileName),
+        ).fileNames,
+    ).not.toContain(testFileName);
+});

src/fileStorage/reducers.ts

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 The Pybricks Authors
+
+import { Reducer, combineReducers } from 'redux';
+import { Action } from '../actions';
+import { FileStorageActionType } from './actions';
+
+const isInitialized: Reducer<boolean, Action> = (state = false, action) => {
+    switch (action.type) {
+        case FileStorageActionType.DidInitialize:
+            return true;
+        default:
+            return state;
+    }
+};
+
+const fileNames: Reducer<Set<string>, Action> = (state = new Set(), action) => {
+    switch (action.type) {
+        case FileStorageActionType.DidChangeItem:
+            return new Set([...state, action.fileName]);
+        case FileStorageActionType.DidRemoveItem:
+            return new Set([...state].filter((value) => value !== action.fileName));
+        default:
+            return state;
+    }
+};
+
+export default combineReducers({ isInitialized, fileNames });

src/fileStorage/sagas.test.ts

@@ -0,0 +1,85 @@
+import { AsyncSaga } from '../../test';
+import {
+    FileStorageActionType,
+    fileStorageDidChangeItem,
+    fileStorageDidInitialize,
+    fileStorageDidReadFile,
+    fileStorageDidWriteFile,
+    fileStorageReadFile,
+    fileStorageWriteFile,
+} from './actions';
+import fileStorage from './sagas';
+
+beforeEach(() => {
+    // localForge uses localStorage as backend in test environment, so we need
+    // to start with a clean slate in each test
+    localStorage.clear();
+});
+
+it('should migrate old program from local storage during initialization', async () => {
+    const oldProgramKey = 'program';
+    const oldProgramContents = '# test program';
+
+    // add item to localStorage to simulate an existing program
+    localStorage.setItem(oldProgramKey, oldProgramContents);
+    expect(localStorage.getItem(oldProgramKey)).toBe(oldProgramContents);
+
+    const saga = new AsyncSaga(fileStorage);
+
+    // initialization should remove the localStorage entry
+    let action = await saga.take();
+    expect(action).toEqual(fileStorageDidInitialize());
+    expect(localStorage.getItem(oldProgramKey)).toBeNull();
+
+    // and add it to the new storage backend
+    action = await saga.take();
+    expect(action).toEqual(fileStorageDidChangeItem('main.py'));
+
+    await saga.end();
+});
+
+it('should read and write files', async () => {
+    const saga = new AsyncSaga(fileStorage);
+
+    let action = await saga.take();
+
+    expect(action).toEqual(fileStorageDidInitialize());
+
+    const testFileName = 'test.file';
+    const testFileContents = 'test file contents';
+
+    // test writing a file
+    saga.put(fileStorageWriteFile(testFileName, testFileContents));
+
+    // writing file triggers response
+    action = await saga.take();
+    expect(action).toEqual(fileStorageDidWriteFile(testFileName));
+
+    // and as a side-effect, triggers item change as well
+    action = await saga.take();
+    expect(action).toEqual(fileStorageDidChangeItem(testFileName));
+
+    // test reading the same file back
+    saga.put(fileStorageReadFile(testFileName));
+
+    action = await saga.take();
+    expect(action).toEqual(fileStorageDidReadFile(testFileName, testFileContents));
+
+    await saga.end();
+});
+
+it('should dispatch fail action if file does not exist', async () => {
+    const saga = new AsyncSaga(fileStorage);
+
+    let action = await saga.take();
+    expect(action).toEqual(fileStorageDidInitialize());
+
+    const testFileName = 'test.file';
+
+    saga.put(fileStorageReadFile(testFileName));
+
+    action = await saga.take();
+    expect(action).toHaveProperty('type', FileStorageActionType.DidFailToReadFile);
+
+    await saga.end();
+});

src/fileStorage/sagas.ts

@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 The Pybricks Authors
+
+import localForage from 'localforage';
+import { extendPrototype } from 'localforage-observable';
+import { eventChannel } from 'redux-saga';
+import { call, fork, put, takeEvery } from 'typed-redux-saga/macro';
+import Observable from 'zen-observable';
+import { ensureError } from '../utils';
+import {
+    FileStorageActionType,
+    FileStorageReadFileAction,
+    FileStorageWriteFileAction,
+    fileStorageDidChangeItem,
+    fileStorageDidFailToInitialize,
+    fileStorageDidFailToReadFile,
+    fileStorageDidFailToWriteFile,
+    fileStorageDidInitialize,
+    fileStorageDidReadFile,
+    fileStorageDidRemoveItem,
+    fileStorageDidWriteFile,
+} from './actions';
+
+/**
+ * Converts localForage change events to redux actions.
+ * @param change The storage change event.
+ */
+function* handleFileStorageDidChange(change: LocalForageObservableChange): Generator {
+    switch (change.methodName) {
+        case 'setItem':
+            if (change.success) {
+                yield* put(fileStorageDidChangeItem(change.key));
+            }
+            break;
+        case 'removeItem':
+            if (change.success) {
+                yield* put(fileStorageDidRemoveItem(change.key));
+            }
+            break;
+    }
+}
+
+/**
+ * Handles requests to read a file.
+ * @param files The storage instance.
+ * @param action The requested action.
+ */
+function* handleReadFile(
+    files: LocalForage,
+    action: FileStorageReadFileAction,
+): Generator {
+    try {
+        const value = yield* call(() => files.getItem<string>(action.fileName));
+
+        if (value === null) {
+            throw new Error('file does not exist');
+        }
+
+        yield* put(fileStorageDidReadFile(action.fileName, value));
+    } catch (err) {
+        yield* put(fileStorageDidFailToReadFile(action.fileName, ensureError(err)));
+    }
+}
+
+/**
+ * Saves the file contents to storage.
+ * @param files The localForage instance.
+ * @param action The action that triggered this saga.
+ */
+function* handleWriteFile(files: LocalForage, action: FileStorageWriteFileAction) {
+    try {
+        yield* call(() => files.setItem(action.fileName, action.fileContents));
+        yield* put(fileStorageDidWriteFile(action.fileName));
+    } catch (err) {
+        yield* put(fileStorageDidFailToWriteFile(action.fileName, ensureError(err)));
+    }
+}
+
+/**
+ * Initializes the storage backend.
+ */
+function* initialize(): Generator {
+    try {
+        // set up storage
+
+        const files = extendPrototype(
+            localForage.createInstance({ name: 'fileStorage' }),
+        );
+
+        files.newObservable.factory = (subscribe) =>
+            // @ts-expect-error localforage-observable Subscription is missing
+            // closed property compared to zen-observable Subscription.
+            new Observable(subscribe);
+
+        yield* call(() => files.ready());
+
+        files.configObservables({
+            crossTabNotification: true,
+            crossTabChangeDetection: true,
+        });
+
+        // wire storage observable to redux-sagas
+
+        const localForageChannel = eventChannel<LocalForageObservableChange>((emit) => {
+            const filesObservable = files.newObservable({
+                crossTabNotification: true,
+            });
+
+            const subscription = filesObservable.subscribe({
+                next: (value) => emit(value),
+            });
+
+            return () => subscription.unsubscribe();
+        });
+
+        // subscribe to events
+
+        yield* takeEvery(localForageChannel, handleFileStorageDidChange);
+        yield* takeEvery(FileStorageActionType.ReadFile, handleReadFile, files);
+        yield* takeEvery(FileStorageActionType.WriteFile, handleWriteFile, files);
+
+        // migrate from old storage
+
+        // Previous versions of pybricks code used local storage to save a single program.
+        const oldProgram = localStorage.getItem('program');
+
+        if (oldProgram !== null) {
+            yield* call(() => files.setItem('main.py', oldProgram));
+            localStorage.removeItem('program');
+        }
+
+        yield* put(fileStorageDidInitialize());
+    } catch (err) {
+        yield* put(fileStorageDidFailToInitialize(ensureError(err)));
+    }
+}
+
+export default function* (): Generator {
+    yield* fork(initialize);
+}