From f788654ff7d575a30cddae21f045e03a5541500f Mon Sep 17 00:00:00 2001 From: garrettmills Date: Wed, 28 Oct 2020 23:48:46 -0500 Subject: [PATCH] Add ability to prefetch and auto-prefetch offline data --- src/app/app.component.ts | 85 ++++++++ .../option-picker.component.html | 8 + .../option-picker/option-picker.component.ts | 10 +- src/app/service/api.service.ts | 186 +++++++++++++++++- src/app/service/db/DatabaseEntry.ts | 2 +- src/app/service/db/Model.ts | 14 +- src/app/service/db/Page.ts | 6 +- src/app/service/db/database.service.ts | 21 ++ src/app/service/editor.service.ts | 18 +- 9 files changed, 332 insertions(+), 18 deletions(-) diff --git a/src/app/app.component.ts b/src/app/app.component.ts index b5b423a..7afb6c4 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -168,6 +168,9 @@ export class AppComponent implements OnInit { toggleDark: () => this.toggleDark(), isDark: () => this.isDark(), showSearch: () => this.handleKeyboardEvent(), + isPrefetch: () => this.isPrefetch(), + togglePrefetch: () => this.togglePrefetch(), + doPrefetch: () => this.doPrefetch(), } }).then(popover => popover.present()); } @@ -472,6 +475,42 @@ export class AppComponent implements OnInit { await this.statusBar.styleDefault(); await this.splashScreen.hide(); + + // If we went online after being offline, sync the local data + if ( !this.api.isOffline && await this.api.needsSync() ) { + this.loader.message = 'Syncing data...'; + try { + await this.api.syncOfflineData(); + } catch (e) { + this.toasts.create({ + cssClass: 'compat-toast-container', + message: 'An error occurred while syncing offline data. Not all data was saved.', + buttons: [ + 'Okay' + ], + }).then(tst => { + tst.present(); + }); + } + } + + if ( this.isPrefetch() ) { + this.loader.message = 'Downloading data...'; // TODO actually do the prefetch + try { + await this.api.prefetchOfflineData(); + } catch (e) { + this.toasts.create({ + cssClass: 'compat-toast-container', + message: 'An error occurred while pre-fetching offline data. Not all data was saved.', + buttons: [ + 'Okay' + ], + }).then(tst => { + tst.present(); + }); + } + } + this.initialized$.next(true); if ( this.session.get('user.preferences.default_page') ) { @@ -483,6 +522,44 @@ export class AppComponent implements OnInit { } } + async doPrefetch() { + if ( this.api.isOffline ) { + return; + } + + this.loader = await this.loading.create({ + message: 'Pre-fetching data...', + cssClass: 'noded-loading-mask', + showBackdrop: true, + }); + + await new Promise(res => setTimeout(res, 2000)); + + await this.loader.present(); + + try { + if (await this.api.needsSync()) { + this.loader.message = 'Syncing data...'; + await this.api.syncOfflineData(); + } + + this.loader.message = 'Downloading data...'; + await this.api.prefetchOfflineData(); + } catch (e) { + const msg = await this.alerts.create({ + header: 'Uh, oh!', + message: 'An unexpected error occurred while trying to sync offline data, and we were unable to continue.', + buttons: [ + 'OK', + ], + }); + + await msg.present(); + } + + this.loader.dismiss(); + } + toggleDark() { // const prefersDark = window.matchMedia('(prefers-color-scheme: dark)'); this.darkMode = !this.darkMode; @@ -490,6 +567,10 @@ export class AppComponent implements OnInit { document.body.classList.toggle('dark', this.darkMode); } + togglePrefetch() { + this.session.set('user.preferences.auto_prefetch', !this.isPrefetch()); + } + findNode(id: string, nodes = this.nodes) { for ( const node of nodes ) { if ( node.id === id ) { @@ -508,4 +589,8 @@ export class AppComponent implements OnInit { isDark() { return !!this.darkMode; } + + isPrefetch() { + return !!this.session.get('user.preferences.auto_prefetch'); + } } diff --git a/src/app/components/option-picker/option-picker.component.html b/src/app/components/option-picker/option-picker.component.html index 9bfb0f2..9d14b34 100644 --- a/src/app/components/option-picker/option-picker.component.html +++ b/src/app/components/option-picker/option-picker.component.html @@ -11,6 +11,14 @@ {{ isDark() ? 'To The Light!' : 'Go Dark...' }} + + + Prefetch Offline Data + + + + Auto-Prefetch + Logout diff --git a/src/app/components/option-picker/option-picker.component.ts b/src/app/components/option-picker/option-picker.component.ts index 179e1ec..b4338dd 100644 --- a/src/app/components/option-picker/option-picker.component.ts +++ b/src/app/components/option-picker/option-picker.component.ts @@ -2,7 +2,7 @@ import {Component, Input, OnInit} from '@angular/core'; import {Router} from '@angular/router'; import {ApiService} from '../../service/api.service'; import {PopoverController} from '@ionic/angular'; -import {DatabaseService} from "../../service/db/database.service"; +import {DatabaseService} from '../../service/db/database.service'; @Component({ selector: 'app-option-picker', @@ -13,6 +13,9 @@ export class OptionPickerComponent implements OnInit { @Input() toggleDark: () => void; @Input() isDark: () => boolean; @Input() showSearch: () => void | Promise; + @Input() isPrefetch: () => boolean; + @Input() togglePrefetch: () => void; + @Input() doPrefetch: () => any; constructor( protected api: ApiService, @@ -34,6 +37,11 @@ export class OptionPickerComponent implements OnInit { } else if ( key === 'search_everywhere' ) { this.showSearch(); this.popover.dismiss(); + } else if ( key === 'toggle_auto_prefetch' ) { + this.togglePrefetch(); + } else if ( key === 'prefetch_data' ) { + this.popover.dismiss(); + this.doPrefetch(); } } } diff --git a/src/app/service/api.service.ts b/src/app/service/api.service.ts index 68485a6..16d9ab9 100644 --- a/src/app/service/api.service.ts +++ b/src/app/service/api.service.ts @@ -11,6 +11,8 @@ import {Database} from './db/Database'; import {DatabaseColumn} from './db/DatabaseColumn'; import {DatabaseEntry} from './db/DatabaseEntry'; import {FileGroup} from './db/FileGroup'; +import {Page} from "./db/Page"; +import {PageNode} from "./db/PageNode"; export class ResourceNotAvailableOfflineError extends Error { constructor(msg = 'This resource is not yet available offline on this device.') { @@ -18,6 +20,12 @@ export class ResourceNotAvailableOfflineError extends Error { } } +export class OfflinePrefetchWouldOverwriteLocalDataError extends Error { + constructor(msg = 'Cannot run offline prefetch. There is un-synced local data that would be overwritten.') { + super(msg); + } +} + @Injectable({ providedIn: 'root' }) @@ -71,12 +79,21 @@ export class ApiService { }); } - public makeOffline() { + public async makeOffline() { + const offlineKV = await this.db.getKeyValue('needs_online_sync'); + offlineKV.data = true; + await offlineKV.save(); + this.offline = true; this.offline$.next(true); } - public makeOnline() { + public async needsSync() { + const offlineKV = await this.db.getKeyValue('needs_online_sync'); + return Boolean(offlineKV.data); + } + + public async makeOnline() { this.offline = false; this.offline$.next(false); } @@ -183,6 +200,167 @@ export class ApiService { return `${this.baseEndpoint.endsWith('/') ? this.baseEndpoint.slice(0, -1) : this.baseEndpoint}${endpoint}`; } + public async syncOfflineData() { + const dirtyRecords = await this.db.getDirtyRecords(); + await new Promise(async (res, rej) => { + this.post('/offline/sync', { dirtyRecords }).subscribe({ + next: async result => { + console.log('sync result', result); + res(); + }, + error: rej, + }); + }); + } + + public async prefetchOfflineData(overwriteLocalData = false) { + if ( this.isOffline ) { + throw new ResourceNotAvailableOfflineError(); + } + + if ( !overwriteLocalData && await this.needsSync() ) { + throw new OfflinePrefetchWouldOverwriteLocalDataError(); + } + + const data: any = await new Promise((res, rej) => { + this.get('/offline/prefetch').subscribe({ + next: result => { + if ( !result.data ) { + return rej(new TypeError('Unable to parse data from API response.')); + } + + res(result.data); + }, + error: rej, + }); + }); + + if ( Array.isArray(data.pages) ) { + await this.db.pages.clear(); + + for ( const rec of data.pages ) { + const page = new Page( + rec.UUID, + rec.Name, + rec.OrgUserId, + rec.IsPublic, + rec.IsVisibleInMenu, + rec.ParentId, + rec.NodeIds, + rec.CreatedAt, + rec.UpdatedAt, + rec.Active, + rec.CreatedUserId, + rec.UpdateUserId, + rec.ChildPageIds, + rec.noDelete, + rec.virtual + ); + + await page.save(); + } + } + + if ( Array.isArray(data.pageNodes) ) { + await this.db.pageNodes.clear(); + + for ( const rec of data.pageNodes ) { + const node = new PageNode( + rec.UUID, + rec.Type, + rec.Value ? JSON.stringify(rec.Value) : JSON.stringify({}), + rec.PageId, + rec.CreatedAt, + rec.UpdatedAt, + rec.CreatedUserId, + rec.UpdateUserId + ); + + await node.save(); + } + } + + if ( Array.isArray(data.codiums) ) { + await this.db.codiums.clear(); + + for ( const rec of data.codiums ) { + const code = new Codium( + rec.Language, + rec.NodeId, + rec.PageId, + rec.code, + rec.UUID + ); + + await code.save(); + } + } + + if ( Array.isArray(data.databases) ) { + await this.db.databases.clear(); + + for ( const rec of data.databases ) { + const db = new Database( + rec.Name, + rec.NodeId, + rec.PageId, + rec.ColumnIds, + rec.UUID, + rec.Active + ); + + await db.save(); + } + } + + if ( Array.isArray(data.databaseColumns) ) { + await this.db.databaseColumns.clear(); + + for ( const rec of data.databaseColumns ) { + const col = new DatabaseColumn( + rec.headerName, + rec.field, + rec.DatabaseId, + rec.UUID, + rec.Type, + rec.additionalData + ); + + await col.save(); + } + } + + if ( Array.isArray(data.databaseEntries) ) { + await this.db.databaseEntries.clear(); + + for ( const rec of data.databaseEntries ) { + const entry = new DatabaseEntry( + rec.DatabaseId, + JSON.stringify(rec.RowData || {}), + rec.UUID + ); + + await entry.save(); + } + } + + if ( Array.isArray(data.fileGroups) ) { + await this.db.fileGroups.clear(); + + for ( const rec of data.fileGroups ) { + const group = new FileGroup( + rec.NodeId, + rec.PageId, + rec.FileIds, + JSON.stringify(rec.files || []), + rec.UUID + ); + + await group.save(); + } + } + } + public getDeviceToken(): Promise { return new Promise(async (res, rej) => { const tokenKV = await this.db.getKeyValue('device_token'); @@ -677,7 +855,8 @@ export class ApiService { const newDatabaseEntry = new DatabaseEntry( row.DatabaseId, JSON.stringify(row.RowData), - row.UUID || DatabaseEntry.getUUID() + row.UUID || DatabaseEntry.getUUID(), + true ); await newDatabaseEntry.save(); @@ -728,6 +907,7 @@ export class ApiService { def.UUID || DatabaseColumn.getUUID(), def.Type, def.additionalData, + true ); await newColumnDef.save(); diff --git a/src/app/service/db/DatabaseEntry.ts b/src/app/service/db/DatabaseEntry.ts index e467ed9..660c8c0 100644 --- a/src/app/service/db/DatabaseEntry.ts +++ b/src/app/service/db/DatabaseEntry.ts @@ -60,7 +60,7 @@ export class DatabaseEntry extends Model implements IDatabaseEnt public inflateToRecord() { const record = this.getSaveRecord(); - record.RowData = JSON.parse(record.RowDataJSON); + record.RowData = record.RowDataJSON ? JSON.parse(record.RowDataJSON) : {}; delete record.RowDataJSON; return record; } diff --git a/src/app/service/db/Model.ts b/src/app/service/db/Model.ts index 2f1f5fe..de5a041 100644 --- a/src/app/service/db/Model.ts +++ b/src/app/service/db/Model.ts @@ -31,6 +31,18 @@ export abstract class Model { } public async save() { - this.id = await this.getDatabase().put(this.getSaveRecord()); + const record = this.getSaveRecord(); + + for ( const prop in record ) { + if ( !record.hasOwnProperty(prop) ) { + continue; + } + + if ( [true, false].includes(record[prop]) ) { + record[prop] = record[prop] ? 1 : 0; + } + } + + this.id = await this.getDatabase().put(record); } } diff --git a/src/app/service/db/Page.ts b/src/app/service/db/Page.ts index 5221b99..1bdb040 100644 --- a/src/app/service/db/Page.ts +++ b/src/app/service/db/Page.ts @@ -17,7 +17,7 @@ export interface IPage { ChildPageIds: string[]; noDelete: boolean; virtual: boolean; - needsServerUpdate?: boolean; + needsServerUpdate?: 0 | 1; deleted?: boolean; } @@ -38,7 +38,7 @@ export class Page extends Model implements IPage { ChildPageIds: string[]; noDelete: boolean; virtual: boolean; - needsServerUpdate?: boolean; + needsServerUpdate?: 0 | 1; deleted?: boolean; public static getTableName() { @@ -66,7 +66,7 @@ export class Page extends Model implements IPage { ChildPageIds: string[], noDelete: boolean, virtual: boolean, - needsServerUpdate?: boolean, + needsServerUpdate?: 0 | 1, deleted?: boolean, id?: number ) { diff --git a/src/app/service/db/database.service.ts b/src/app/service/db/database.service.ts index 31b162d..6d7310b 100644 --- a/src/app/service/db/database.service.ts +++ b/src/app/service/db/database.service.ts @@ -97,6 +97,27 @@ export class DatabaseService extends Dexie { this.pageNodes.mapToClass(PageNode); } + /** Return the local database records that need to be synced up with the server. */ + public async getDirtyRecords() { + const codiums = await this.codiums.where({ needsServerUpdate: 1 }).toArray() as Codium[]; + const databases = await this.databases.where({ needsServerUpdate: 1 }).toArray() as Database[]; + const databaseColumns = await this.databaseColumns.where({ needsServerUpdate: 1 }).toArray() as DatabaseColumn[]; + const databaseEntries = await this.databaseEntries.where({ needsServerUpdate: 1 }).toArray() as DatabaseEntry[]; + const fileGroups = await this.fileGroups.where({ needsServerUpdate: 1 }).toArray() as FileGroup[]; + const pages = await this.pages.where({ needsServerUpdate: 1 }).toArray() as Page[]; + const pageNodes = await this.pageNodes.where({ needsServerUpdate: 1 }).toArray() as PageNode[]; + + return { + codiums: codiums.map(x => x.getSaveRecord()), + databases: databases.map(x => x.getSaveRecord()), + databaseColumns: databaseColumns.map(x => x.getSaveRecord()), + databaseEntries: databaseEntries.map(x => x.getSaveRecord()), + fileGroups: fileGroups.map(x => x.getSaveRecord()), + pages: pages.map(x => x.getSaveRecord()), + pageNodes: pageNodes.map(x => x.getSaveRecord()), + }; + } + public async purge() { console.warn('Purging all local data!'); diff --git a/src/app/service/editor.service.ts b/src/app/service/editor.service.ts index 0bcfafc..0310bcc 100644 --- a/src/app/service/editor.service.ts +++ b/src/app/service/editor.service.ts @@ -177,7 +177,7 @@ export class EditorService { if ( existingLocalPage ) { existingLocalPage.fillFromRecord(page); existingLocalPage.UpdatedAt = String(new Date()); // FIXME update UpdateUserId - existingLocalPage.needsServerUpdate = true; + existingLocalPage.needsServerUpdate = 1; await existingLocalPage.save(); } else { const newLocalPage = new Page( @@ -196,7 +196,7 @@ export class EditorService { page.ChildPageIds, false, false, - true, + 1, ); await newLocalPage.save(); @@ -227,7 +227,7 @@ export class EditorService { result.data.ChildPageIds, result.data.noDelete, result.data.virtual, - false + 0 ); await newLocalPage.save(); @@ -283,7 +283,7 @@ export class EditorService { page.NodeIds = nodeRecs.map(x => x.UUID); existingLocalPage.NodeIds = nodeRecs.map(x => x.UUID); - existingLocalPage.needsServerUpdate = true; + existingLocalPage.needsServerUpdate = 1; await existingLocalPage.save(); return res(nodeRecs.map(x => { @@ -356,7 +356,7 @@ export class EditorService { await newLocalNode.save(); localPage.NodeIds.push(newLocalNode.UUID); - localPage.needsServerUpdate = true; + localPage.needsServerUpdate = 1; await localPage.save(); const host = new HostRecord(nodeData.Value.Value); @@ -446,7 +446,7 @@ export class EditorService { baseHost.PageId = this.currentPage.UUID; const host = await this.saveNodeToPage(this.currentPage, baseHost); - console.log('added node to page', host) + console.log('added node to page', host); let placed = false; if ( position === 'before' && positionNodeId ) { @@ -542,7 +542,7 @@ export class EditorService { if ( existingLocalPage ) { existingLocalPage.fillFromRecord(result.data); - existingLocalPage.needsServerUpdate = false; + existingLocalPage.needsServerUpdate = 0; await existingLocalPage.save(); } else { const newLocalPage = new Page( @@ -643,13 +643,13 @@ export class EditorService { [], false, false, - true + 1 ); const firstNode = new PageNode( PageNode.getUUID(), 'paragraph', - JSON.stringify({ Value: 'Click to edit...' }), + JSON.stringify({ Value: 'Double-click to edit...' }), page.UUID, String(new Date()), String(new Date()),