diff --git a/app/gen-server/entity/User.ts b/app/gen-server/entity/User.ts index 2ed10169..c93837cb 100644 --- a/app/gen-server/entity/User.ts +++ b/app/gen-server/entity/User.ts @@ -29,6 +29,9 @@ export class User extends BaseEntity { @Column({name: 'first_login_at', type: Date, nullable: true}) public firstLoginAt: Date | null; + @Column({name: 'last_connection_at', type: Date, nullable: true}) + public lastConnectionAt: Date | null; + @OneToOne(type => Organization, organization => organization.owner) public personalOrg: Organization; diff --git a/app/gen-server/lib/homedb/UsersManager.ts b/app/gen-server/lib/homedb/UsersManager.ts index 8c0a5dca..168665f3 100644 --- a/app/gen-server/lib/homedb/UsersManager.ts +++ b/app/gen-server/lib/homedb/UsersManager.ts @@ -395,14 +395,6 @@ export class UsersManager { user.name = (profile && (profile.name || email.split('@')[0])) || ''; needUpdate = true; } - if (profile && !user.firstLoginAt) { - // set first login time to now (remove milliseconds for compatibility with other - // timestamps in db set by typeorm, and since second level precision is fine) - const nowish = new Date(); - nowish.setMilliseconds(0); - user.firstLoginAt = nowish; - needUpdate = true; - } if (!user.picture && profile && profile.picture) { // Set the user's profile picture if our provider knows it. user.picture = profile.picture; @@ -432,6 +424,25 @@ export class UsersManager { user.options = {...(user.options ?? {}), authSubject: userOptions.authSubject}; needUpdate = true; } + + // get date of now (remove milliseconds for compatibility with other + // timestamps in db set by typeorm, and since second level precision is fine) + const nowish = new Date(); + nowish.setMilliseconds(0); + if (profile && !user.firstLoginAt) { + // set first login time to now + user.firstLoginAt = nowish; + needUpdate = true; + } + const getTimestampStartOfDay = (date: Date) => { + const timestamp = Math.floor(date.getTime() / 1000); // unix timestamp seconds from epoc + const startOfDay = timestamp - (timestamp % 86400 /*24h*/); // start of a day in seconds since epoc + return startOfDay; + }; + if (!user.lastConnectionAt || getTimestampStartOfDay(user.lastConnectionAt) !== getTimestampStartOfDay(nowish)) { + user.lastConnectionAt = nowish; + needUpdate = true; + } if (needUpdate) { login.user = user; await manager.save([user, login]); diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index ba0e71b1..60c86668 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,5 +1,5 @@ -import {User} from 'app/gen-server/entity/User'; import {makeId} from 'app/server/lib/idUtils'; +import {chunk} from 'lodash'; import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; export class UserUUID1663851423064 implements MigrationInterface { @@ -16,11 +16,20 @@ export class UserUUID1663851423064 implements MigrationInterface { // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks. // 300 seems to be a good number, for 24k rows we have 80 queries. const userList = await queryRunner.manager.createQueryBuilder() - .select("users") - .from(User, "users") + .select(["users.id", "users.ref"]) + .from("users", "users") .getMany(); userList.forEach(u => u.ref = makeId()); - await queryRunner.manager.save(userList, { chunk: 300 }); + + const userChunks = chunk(userList, 300); + for (const users of userChunks) { + await queryRunner.connection.transaction(async manager => { + const queries = users.map((user: any, _index: number, _array: any[]) => { + return queryRunner.manager.update("users", user.id, user); + }); + await Promise.all(queries); + }); + } // We are not making this column unique yet, because it can fail // if there are some old workers still running, and any new user diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index 27536042..149be01e 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,5 +1,5 @@ -import {User} from 'app/gen-server/entity/User'; import {makeId} from 'app/server/lib/idUtils'; +import {chunk} from 'lodash'; import {MigrationInterface, QueryRunner} from "typeorm"; export class UserRefUnique1664528376930 implements MigrationInterface { @@ -9,12 +9,21 @@ export class UserRefUnique1664528376930 implements MigrationInterface { // Update users that don't have unique ref set. const userList = await queryRunner.manager.createQueryBuilder() - .select("users") - .from(User, "users") - .where("ref is null") - .getMany(); + .select(["users.id", "users.ref"]) + .from("users", "users") + .where("users.ref is null") + .getMany(); userList.forEach(u => u.ref = makeId()); - await queryRunner.manager.save(userList, {chunk: 300}); + + const userChunks = chunk(userList, 300); + for (const users of userChunks) { + await queryRunner.connection.transaction(async manager => { + const queries = users.map((user: any, _index: number, _array: any[]) => { + return queryRunner.manager.update("users", user.id, user); + }); + await Promise.all(queries); + }); + } // Mark column as unique and non-nullable. const users = (await queryRunner.getTable('users'))!; diff --git a/app/gen-server/migration/1713186031023-UserLastConnection.ts b/app/gen-server/migration/1713186031023-UserLastConnection.ts new file mode 100644 index 00000000..52310a38 --- /dev/null +++ b/app/gen-server/migration/1713186031023-UserLastConnection.ts @@ -0,0 +1,18 @@ +import {MigrationInterface, QueryRunner, TableColumn} from 'typeorm'; + +export class UserLastConnection1713186031023 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; + const datetime = sqlite ? "datetime" : "timestamp with time zone"; + await queryRunner.addColumn('users', new TableColumn({ + name: 'last_connection_at', + type: datetime, + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('users', 'last_connection_at'); + } +} diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index a6f29106..de0326d4 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -21,8 +21,8 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ? // Database fields that we permit in entities but don't want to cross the api. const INTERNAL_FIELDS = new Set([ - 'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId', - 'stripeSubscriptionId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin', + 'apiKey', 'billingAccountId', 'firstLoginAt', 'lastConnectionAt', 'filteredOut', 'ownerId', 'gracePeriodStart', + 'stripeCustomerId', 'stripeSubscriptionId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin', 'authSubject', 'usage', 'createdBy' ]); diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts index 9b3e31e5..e6a45b98 100644 --- a/test/gen-server/migrations.ts +++ b/test/gen-server/migrations.ts @@ -42,6 +42,8 @@ import {ActivationPrefs1682636695021 as ActivationPrefs} from 'app/gen-server/mi import {AssistantLimit1685343047786 as AssistantLimit} from 'app/gen-server/migration/1685343047786-AssistantLimit'; import {Shares1701557445716 as Shares} from 'app/gen-server/migration/1701557445716-Shares'; import {Billing1711557445716 as BillingFeatures} from 'app/gen-server/migration/1711557445716-Billing'; +import {UserLastConnection1713186031023 + as UserLastConnection} from 'app/gen-server/migration/1713186031023-UserLastConnection'; const home: HomeDBManager = new HomeDBManager(); @@ -50,7 +52,8 @@ const migrations = [Initial, Login, PinDocs, UserPicture, DisplayEmail, DisplayE CustomerIndex, ExtraIndexes, OrgHost, DocRemovedAt, Prefs, ExternalBilling, DocOptions, Secret, UserOptions, GracePeriodStart, DocumentUsage, Activations, UserConnectId, UserUUID, UserUniqueRefUUID, - Forks, ForkIndexes, ActivationPrefs, AssistantLimit, Shares, BillingFeatures]; + Forks, ForkIndexes, ActivationPrefs, AssistantLimit, Shares, BillingFeatures, + UserLastConnection]; // Assert that the "members" acl rule and group exist (or not). function assertMembersGroup(org: Organization, exists: boolean) { @@ -113,6 +116,33 @@ describe('migrations', function() { // be doing something. }); + it('can migrate UserUUID and UserUniqueRefUUID with user in table', async function() { + this.timeout(60000); + const runner = home.connection.createQueryRunner(); + + // Create 400 users to test the chunk (each chunk is 300 users) + const nbUsersToCreate = 400; + for (const migration of migrations) { + if (migration === UserUUID) { + for (let i = 0; i < nbUsersToCreate; i++) { + await runner.query(`INSERT INTO users (id, name, is_first_time_user) VALUES (${i}, 'name${i}', true)`); + } + } + + await (new migration()).up(runner); + } + + // Check that all refs are unique + const userList = await runner.manager.createQueryBuilder() + .select(["users.id", "users.ref"]) + .from("users", "users") + .getMany(); + const setOfUserRefs = new Set(userList.map(u => u.ref)); + assert.equal(nbUsersToCreate, userList.length); + assert.equal(setOfUserRefs.size, userList.length); + await addSeedData(home.connection); + }); + it('can correctly switch display_email column to non-null with data', async function() { this.timeout(60000); const sqlite = home.connection.driver.options.type === 'sqlite';