Add ability to prefetch and auto-prefetch offline data

src/app/app.component.ts

@@ -168,6 +168,9 @@ export class AppComponent implements OnInit {
         toggleDark: () => this.toggleDark(),
         isDark: () => this.isDark(),
         showSearch: () => this.handleKeyboardEvent(),
+        isPrefetch: () => this.isPrefetch(),
+        togglePrefetch: () => this.togglePrefetch(),
+        doPrefetch: () => this.doPrefetch(),
       }
     }).then(popover => popover.present());
   }
@@ -472,6 +475,42 @@ export class AppComponent implements OnInit {
 
     await this.statusBar.styleDefault();
     await this.splashScreen.hide();
+
+    // If we went online after being offline, sync the local data
+    if ( !this.api.isOffline && await this.api.needsSync() ) {
+      this.loader.message = 'Syncing data...';
+      try {
+        await this.api.syncOfflineData();
+      } catch (e) {
+        this.toasts.create({
+          cssClass: 'compat-toast-container',
+          message: 'An error occurred while syncing offline data. Not all data was saved.',
+          buttons: [
+            'Okay'
+          ],
+        }).then(tst => {
+          tst.present();
+        });
+      }
+    }
+
+    if ( this.isPrefetch() ) {
+      this.loader.message = 'Downloading data...'; // TODO actually do the prefetch
+      try {
+        await this.api.prefetchOfflineData();
+      } catch (e) {
+        this.toasts.create({
+          cssClass: 'compat-toast-container',
+          message: 'An error occurred while pre-fetching offline data. Not all data was saved.',
+          buttons: [
+              'Okay'
+          ],
+        }).then(tst => {
+          tst.present();
+        });
+      }
+    }
+
     this.initialized$.next(true);
 
     if ( this.session.get('user.preferences.default_page') ) {
@@ -483,6 +522,44 @@ export class AppComponent implements OnInit {
     }
   }
 
+  async doPrefetch() {
+    if ( this.api.isOffline ) {
+      return;
+    }
+
+    this.loader = await this.loading.create({
+      message: 'Pre-fetching data...',
+      cssClass: 'noded-loading-mask',
+      showBackdrop: true,
+    });
+
+    await new Promise(res => setTimeout(res, 2000));
+
+    await this.loader.present();
+
+    try {
+      if (await this.api.needsSync()) {
+        this.loader.message = 'Syncing data...';
+        await this.api.syncOfflineData();
+      }
+
+      this.loader.message = 'Downloading data...';
+      await this.api.prefetchOfflineData();
+    } catch (e) {
+      const msg = await this.alerts.create({
+        header: 'Uh, oh!',
+        message: 'An unexpected error occurred while trying to sync offline data, and we were unable to continue.',
+        buttons: [
+          'OK',
+        ],
+      });
+
+      await msg.present();
+    }
+
+    this.loader.dismiss();
+  }
+
   toggleDark() {
     // const prefersDark = window.matchMedia('(prefers-color-scheme: dark)');
     this.darkMode = !this.darkMode;
@@ -490,6 +567,10 @@ export class AppComponent implements OnInit {
     document.body.classList.toggle('dark', this.darkMode);
   }
 
+  togglePrefetch() {
+    this.session.set('user.preferences.auto_prefetch', !this.isPrefetch());
+  }
+
   findNode(id: string, nodes = this.nodes) {
     for ( const node of nodes ) {
       if ( node.id === id ) {
@@ -508,4 +589,8 @@ export class AppComponent implements OnInit {
   isDark() {
     return !!this.darkMode;
   }
+
+  isPrefetch() {
+    return !!this.session.get('user.preferences.auto_prefetch');
+  }
 }

src/app/components/option-picker/option-picker.component.html

@@ -11,6 +11,14 @@
     <i slot="start" class="fa" [ngClass]="isDark() ? 'fa-sun' : 'fa-moon'"></i>
     <ion-label>{{ isDark() ? 'To The Light!' : 'Go Dark...' }}</ion-label>
   </ion-item>
+  <ion-item button (click)="onSelect('prefetch_data')">
+    <i slot="start" class="fa fa-download"></i>
+    <ion-label>Prefetch Offline Data</ion-label>
+  </ion-item>
+  <ion-item button (click)="onSelect('toggle_auto_prefetch')">
+    <i slot="start" class="fa" [ngClass]="isPrefetch() ? 'fa-check-square' : 'fa-square'"></i>
+    <ion-label>Auto-Prefetch</ion-label>
+  </ion-item>
   <ion-item button (click)="onSelect('logout')">
     <i slot="start" class="fa fa-sign-out-alt"></i>
     <ion-label>Logout</ion-label>

src/app/components/option-picker/option-picker.component.ts

@@ -2,7 +2,7 @@ import {Component, Input, OnInit} from '@angular/core';
 import {Router} from '@angular/router';
 import {ApiService} from '../../service/api.service';
 import {PopoverController} from '@ionic/angular';
-import {DatabaseService} from "../../service/db/database.service";
+import {DatabaseService} from '../../service/db/database.service';
 
 @Component({
   selector: 'app-option-picker',
@@ -13,6 +13,9 @@ export class OptionPickerComponent implements OnInit {
   @Input() toggleDark: () => void;
   @Input() isDark: () => boolean;
   @Input() showSearch: () => void | Promise<void>;
+  @Input() isPrefetch: () => boolean;
+  @Input() togglePrefetch: () => void;
+  @Input() doPrefetch: () => any;
 
   constructor(
       protected api: ApiService,
@@ -34,6 +37,11 @@ export class OptionPickerComponent implements OnInit {
     } else if ( key === 'search_everywhere' ) {
       this.showSearch();
       this.popover.dismiss();
+    } else if ( key === 'toggle_auto_prefetch' ) {
+      this.togglePrefetch();
+    } else if ( key === 'prefetch_data' ) {
+      this.popover.dismiss();
+      this.doPrefetch();
     }
   }
 }

src/app/service/api.service.ts

@@ -11,6 +11,8 @@ import {Database} from './db/Database';
 import {DatabaseColumn} from './db/DatabaseColumn';
 import {DatabaseEntry} from './db/DatabaseEntry';
 import {FileGroup} from './db/FileGroup';
+import {Page} from "./db/Page";
+import {PageNode} from "./db/PageNode";
 
 export class ResourceNotAvailableOfflineError extends Error {
   constructor(msg = 'This resource is not yet available offline on this device.') {
@@ -18,6 +20,12 @@ export class ResourceNotAvailableOfflineError extends Error {
   }
 }
 
+export class OfflinePrefetchWouldOverwriteLocalDataError extends Error {
+  constructor(msg = 'Cannot run offline prefetch. There is un-synced local data that would be overwritten.') {
+    super(msg);
+  }
+}
+
 @Injectable({
   providedIn: 'root'
 })
@@ -71,12 +79,21 @@ export class ApiService {
     });
   }
 
-  public makeOffline() {
+  public async makeOffline() {
+    const offlineKV = await this.db.getKeyValue('needs_online_sync');
+    offlineKV.data = true;
+    await offlineKV.save();
+
     this.offline = true;
     this.offline$.next(true);
   }
 
-  public makeOnline() {
+  public async needsSync() {
+    const offlineKV = await this.db.getKeyValue('needs_online_sync');
+    return Boolean(offlineKV.data);
+  }
+
+  public async makeOnline() {
     this.offline = false;
     this.offline$.next(false);
   }
@@ -183,6 +200,167 @@ export class ApiService {
     return `${this.baseEndpoint.endsWith('/') ? this.baseEndpoint.slice(0, -1) : this.baseEndpoint}${endpoint}`;
   }
 
+  public async syncOfflineData() {
+    const dirtyRecords = await this.db.getDirtyRecords();
+    await new Promise(async (res, rej) => {
+      this.post('/offline/sync', { dirtyRecords }).subscribe({
+        next: async result => {
+          console.log('sync result', result);
+          res();
+        },
+        error: rej,
+      });
+    });
+  }
+
+  public async prefetchOfflineData(overwriteLocalData = false) {
+    if ( this.isOffline ) {
+      throw new ResourceNotAvailableOfflineError();
+    }
+
+    if ( !overwriteLocalData && await this.needsSync() ) {
+      throw new OfflinePrefetchWouldOverwriteLocalDataError();
+    }
+
+    const data: any = await new Promise((res, rej) => {
+      this.get('/offline/prefetch').subscribe({
+        next: result => {
+          if ( !result.data ) {
+            return rej(new TypeError('Unable to parse data from API response.'));
+          }
+
+          res(result.data);
+        },
+        error: rej,
+      });
+    });
+
+    if ( Array.isArray(data.pages) ) {
+      await this.db.pages.clear();
+
+      for ( const rec of data.pages ) {
+        const page = new Page(
+            rec.UUID,
+            rec.Name,
+            rec.OrgUserId,
+            rec.IsPublic,
+            rec.IsVisibleInMenu,
+            rec.ParentId,
+            rec.NodeIds,
+            rec.CreatedAt,
+            rec.UpdatedAt,
+            rec.Active,
+            rec.CreatedUserId,
+            rec.UpdateUserId,
+            rec.ChildPageIds,
+            rec.noDelete,
+            rec.virtual
+        );
+
+        await page.save();
+      }
+    }
+
+    if ( Array.isArray(data.pageNodes) ) {
+      await this.db.pageNodes.clear();
+
+      for ( const rec of data.pageNodes ) {
+        const node = new PageNode(
+            rec.UUID,
+            rec.Type,
+            rec.Value ? JSON.stringify(rec.Value) : JSON.stringify({}),
+            rec.PageId,
+            rec.CreatedAt,
+            rec.UpdatedAt,
+            rec.CreatedUserId,
+            rec.UpdateUserId
+        );
+
+        await node.save();
+      }
+    }
+
+    if ( Array.isArray(data.codiums) ) {
+      await this.db.codiums.clear();
+
+      for ( const rec of data.codiums ) {
+        const code = new Codium(
+            rec.Language,
+            rec.NodeId,
+            rec.PageId,
+            rec.code,
+            rec.UUID
+        );
+
+        await code.save();
+      }
+    }
+
+    if ( Array.isArray(data.databases) ) {
+      await this.db.databases.clear();
+
+      for ( const rec of data.databases ) {
+        const db = new Database(
+            rec.Name,
+            rec.NodeId,
+            rec.PageId,
+            rec.ColumnIds,
+            rec.UUID,
+            rec.Active
+        );
+
+        await db.save();
+      }
+    }
+
+    if ( Array.isArray(data.databaseColumns) ) {
+      await this.db.databaseColumns.clear();
+
+      for ( const rec of data.databaseColumns ) {
+        const col = new DatabaseColumn(
+            rec.headerName,
+            rec.field,
+            rec.DatabaseId,
+            rec.UUID,
+            rec.Type,
+            rec.additionalData
+        );
+
+        await col.save();
+      }
+    }
+
+    if ( Array.isArray(data.databaseEntries) ) {
+      await this.db.databaseEntries.clear();
+
+      for ( const rec of data.databaseEntries ) {
+        const entry = new DatabaseEntry(
+            rec.DatabaseId,
+            JSON.stringify(rec.RowData || {}),
+            rec.UUID
+        );
+
+        await entry.save();
+      }
+    }
+
+    if ( Array.isArray(data.fileGroups) ) {
+      await this.db.fileGroups.clear();
+
+      for ( const rec of data.fileGroups ) {
+        const group = new FileGroup(
+            rec.NodeId,
+            rec.PageId,
+            rec.FileIds,
+            JSON.stringify(rec.files || []),
+            rec.UUID
+        );
+
+        await group.save();
+      }
+    }
+  }
+
   public getDeviceToken(): Promise<string> {
     return new Promise(async (res, rej) => {
       const tokenKV = await this.db.getKeyValue('device_token');
@@ -677,7 +855,8 @@ export class ApiService {
           const newDatabaseEntry = new DatabaseEntry(
               row.DatabaseId,
               JSON.stringify(row.RowData),
-              row.UUID || DatabaseEntry.getUUID()
+              row.UUID || DatabaseEntry.getUUID(),
+              true
           );
 
           await newDatabaseEntry.save();
@@ -728,6 +907,7 @@ export class ApiService {
                 def.UUID || DatabaseColumn.getUUID(),
                 def.Type,
                 def.additionalData,
+                true
             );
 
             await newColumnDef.save();

src/app/service/db/DatabaseEntry.ts

@@ -60,7 +60,7 @@ export class DatabaseEntry extends Model<IDatabaseEntry> implements IDatabaseEnt
 
     public inflateToRecord() {
         const record = this.getSaveRecord();
-        record.RowData = JSON.parse(record.RowDataJSON);
+        record.RowData = record.RowDataJSON ? JSON.parse(record.RowDataJSON) : {};
         delete record.RowDataJSON;
         return record;
     }

src/app/service/db/Model.ts

@@ -31,6 +31,18 @@ export abstract class Model<InterfaceType> {
     }
 
     public async save() {
-        this.id = await this.getDatabase().put(this.getSaveRecord());
+        const record = this.getSaveRecord();
+
+        for ( const prop in record ) {
+            if ( !record.hasOwnProperty(prop) ) {
+                continue;
+            }
+
+            if ( [true, false].includes(record[prop]) ) {
+                record[prop] = record[prop] ? 1 : 0;
+            }
+        }
+
+        this.id = await this.getDatabase().put(record);
     }
 }

src/app/service/db/Page.ts

@@ -17,7 +17,7 @@ export interface IPage {
     ChildPageIds: string[];
     noDelete: boolean;
     virtual: boolean;
-    needsServerUpdate?: boolean;
+    needsServerUpdate?: 0 | 1;
     deleted?: boolean;
 }
 
@@ -38,7 +38,7 @@ export class Page extends Model<IPage> implements IPage {
     ChildPageIds: string[];
     noDelete: boolean;
     virtual: boolean;
-    needsServerUpdate?: boolean;
+    needsServerUpdate?: 0 | 1;
     deleted?: boolean;
 
     public static getTableName() {
@@ -66,7 +66,7 @@ export class Page extends Model<IPage> implements IPage {
         ChildPageIds: string[],
         noDelete: boolean,
         virtual: boolean,
-        needsServerUpdate?: boolean,
+        needsServerUpdate?: 0 | 1,
         deleted?: boolean,
         id?: number
     ) {

src/app/service/db/database.service.ts

@@ -97,6 +97,27 @@ export class DatabaseService extends Dexie {
         this.pageNodes.mapToClass(PageNode);
     }
 
+    /** Return the local database records that need to be synced up with the server. */
+    public async getDirtyRecords() {
+        const codiums = await this.codiums.where({ needsServerUpdate: 1 }).toArray() as Codium[];
+        const databases = await this.databases.where({ needsServerUpdate: 1 }).toArray() as Database[];
+        const databaseColumns = await this.databaseColumns.where({ needsServerUpdate: 1 }).toArray() as DatabaseColumn[];
+        const databaseEntries = await this.databaseEntries.where({ needsServerUpdate: 1 }).toArray() as DatabaseEntry[];
+        const fileGroups = await this.fileGroups.where({ needsServerUpdate: 1 }).toArray() as FileGroup[];
+        const pages = await this.pages.where({ needsServerUpdate: 1 }).toArray() as Page[];
+        const pageNodes = await this.pageNodes.where({ needsServerUpdate: 1 }).toArray() as PageNode[];
+
+        return {
+            codiums: codiums.map(x => x.getSaveRecord()),
+            databases: databases.map(x => x.getSaveRecord()),
+            databaseColumns: databaseColumns.map(x => x.getSaveRecord()),
+            databaseEntries: databaseEntries.map(x => x.getSaveRecord()),
+            fileGroups: fileGroups.map(x => x.getSaveRecord()),
+            pages: pages.map(x => x.getSaveRecord()),
+            pageNodes: pageNodes.map(x => x.getSaveRecord()),
+        };
+    }
+
     public async purge() {
         console.warn('Purging all local data!');
 

src/app/service/editor.service.ts

@@ -177,7 +177,7 @@ export class EditorService {
                 if ( existingLocalPage ) {
                     existingLocalPage.fillFromRecord(page);
                     existingLocalPage.UpdatedAt = String(new Date()); // FIXME update UpdateUserId
-                    existingLocalPage.needsServerUpdate = true;
+                    existingLocalPage.needsServerUpdate = 1;
                     await existingLocalPage.save();
                 } else {
                     const newLocalPage = new Page(
@@ -196,7 +196,7 @@ export class EditorService {
                         page.ChildPageIds,
                         false,
                         false,
-                        true,
+                        1,
                     );
 
                     await newLocalPage.save();
@@ -227,7 +227,7 @@ export class EditorService {
                             result.data.ChildPageIds,
                             result.data.noDelete,
                             result.data.virtual,
-                            false
+                            0
                         );
 
                         await newLocalPage.save();
@@ -283,7 +283,7 @@ export class EditorService {
 
                 page.NodeIds = nodeRecs.map(x => x.UUID);
                 existingLocalPage.NodeIds = nodeRecs.map(x => x.UUID);
-                existingLocalPage.needsServerUpdate = true;
+                existingLocalPage.needsServerUpdate = 1;
                 await existingLocalPage.save();
 
                 return res(nodeRecs.map(x => {
@@ -356,7 +356,7 @@ export class EditorService {
                 await newLocalNode.save();
 
                 localPage.NodeIds.push(newLocalNode.UUID);
-                localPage.needsServerUpdate = true;
+                localPage.needsServerUpdate = 1;
                 await localPage.save();
 
                 const host = new HostRecord(nodeData.Value.Value);
@@ -446,7 +446,7 @@ export class EditorService {
         baseHost.PageId = this.currentPage.UUID;
 
         const host = await this.saveNodeToPage(this.currentPage, baseHost);
-        console.log('added node to page', host)
+        console.log('added node to page', host);
 
         let placed = false;
         if ( position === 'before' && positionNodeId ) {
@@ -542,7 +542,7 @@ export class EditorService {
 
                     if ( existingLocalPage ) {
                         existingLocalPage.fillFromRecord(result.data);
-                        existingLocalPage.needsServerUpdate = false;
+                        existingLocalPage.needsServerUpdate = 0;
                         await existingLocalPage.save();
                     } else {
                         const newLocalPage = new Page(
@@ -643,13 +643,13 @@ export class EditorService {
                     [],
                     false,
                     false,
-                    true
+                    1
                 );
 
                 const firstNode = new PageNode(
                     PageNode.getUUID(),
                     'paragraph',
-                    JSON.stringify({ Value: 'Click to edit...' }),
+                    JSON.stringify({ Value: 'Double-click to edit...' }),
                     page.UUID,
                     String(new Date()),
                     String(new Date()),