From 43f7fbeff49a215a18240714e675f032f441968b Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Sep 2024 16:36:34 -0400 Subject: [PATCH 1/2] Fix DateEditor test (newly moved to grist-core) by adding a sinon.js symlink to static/ (#1206) --- static/sinon.js | 1 + 1 file changed, 1 insertion(+) create mode 120000 static/sinon.js diff --git a/static/sinon.js b/static/sinon.js new file mode 120000 index 00000000..87d998ec --- /dev/null +++ b/static/sinon.js @@ -0,0 +1 @@ +../node_modules/sinon/pkg/sinon.js \ No newline at end of file From a45a7f9fda6d824fafbb190757e73ea27a7e9019 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Fri, 13 Sep 2024 14:56:42 -0400 Subject: [PATCH 2/2] switch default LLM from a model that is going away (#1202) * switch default LLM from a model that is going away If an api key is provided, Grist can use an LLM as an assistant for writing formulas. The LLM can be self-hosted or an external service. The default external service is OpenAI. This commit freshens the default model used, if this feature is enabled, since the existing one is going away. Benchmarking suggests the results are generally better, though not dramatically so. The feature of falling back on a longer context model is no longer as important, but is retained since it could be useful for self-hosters. * update long context model tests --- app/server/lib/Assistance.ts | 4 ++-- test/server/lib/Assistance.ts | 39 +++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/app/server/lib/Assistance.ts b/app/server/lib/Assistance.ts index 875752ad..e57b0e7b 100644 --- a/app/server/lib/Assistance.ts +++ b/app/server/lib/Assistance.ts @@ -148,8 +148,8 @@ class RetryableError extends Error { * An optional ASSISTANT_MAX_TOKENS can be specified. */ export class OpenAIAssistant implements Assistant { - public static DEFAULT_MODEL = "gpt-3.5-turbo-0613"; - public static DEFAULT_LONGER_CONTEXT_MODEL = "gpt-3.5-turbo-16k-0613"; + public static DEFAULT_MODEL = "gpt-4o-2024-08-06"; + public static DEFAULT_LONGER_CONTEXT_MODEL = ""; private _apiKey?: string; private _model?: string; diff --git a/test/server/lib/Assistance.ts b/test/server/lib/Assistance.ts index 6ad8c6cb..4a73aef0 100644 --- a/test/server/lib/Assistance.ts +++ b/test/server/lib/Assistance.ts @@ -1,11 +1,12 @@ -import {createDocTools} from "test/server/docTools"; +import {AssistanceState} from 'app/common/AssistancePrompts'; import {ActiveDoc} from "app/server/lib/ActiveDoc"; -import {DEPS, OpenAIAssistant, sendForCompletion} from "app/server/lib/Assistance"; +import {DEPS, OpenAIAssistant, sendForCompletion} from 'app/server/lib/Assistance'; +import {DocSession} from 'app/server/lib/DocSession'; import {assert} from 'chai'; -import * as sinon from 'sinon'; import {Response} from 'node-fetch'; -import {DocSession} from "app/server/lib/DocSession"; -import {AssistanceState} from "app/common/AssistancePrompts"; +import * as sinon from 'sinon'; +import {createDocTools} from 'test/server/docTools'; +import {EnvironmentSnapshot} from 'test/server/testUtils'; // For some reason, assert.isRejected is not getting defined, // though test/chai-as-promised.js should be taking care of this. @@ -14,6 +15,12 @@ const chai = require('chai'); const chaiAsPromised = require('chai-as-promised'); chai.use(chaiAsPromised); +/** + * We no longer use a longer context model by default, but we still + * test this configuration. + */ +const LONGER_CONTEXT_MODEL_FOR_TEST = "fake"; + describe('Assistance', function () { this.timeout(10000); @@ -22,8 +29,11 @@ describe('Assistance', function () { const table2Id = "Table2"; let session: DocSession; let doc: ActiveDoc; + let oldEnv: EnvironmentSnapshot; before(async () => { + oldEnv = new EnvironmentSnapshot(); process.env.OPENAI_API_KEY = "fake"; + process.env.ASSISTANT_LONGER_CONTEXT_MODEL = LONGER_CONTEXT_MODEL_FOR_TEST; session = docTools.createFakeSession(); doc = await docTools.createDoc('test.grist'); await doc.applyUserActions(session, [ @@ -31,6 +41,9 @@ describe('Assistance', function () { ["AddTable", table2Id, [{id: "A"}, {id: "B"}, {id: "C"}]], ]); }); + after(async function () { + oldEnv.restore(); + }); const colId = "C"; const userMessageContent = "Sum of A and B"; @@ -204,8 +217,8 @@ describe('Assistance', function () { ); checkModels([ OpenAIAssistant.DEFAULT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, + LONGER_CONTEXT_MODEL_FOR_TEST, + LONGER_CONTEXT_MODEL_FOR_TEST, ]); }); @@ -254,8 +267,8 @@ describe('Assistance', function () { ); checkModels([ OpenAIAssistant.DEFAULT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, + LONGER_CONTEXT_MODEL_FOR_TEST, + LONGER_CONTEXT_MODEL_FOR_TEST, ]); }); @@ -278,8 +291,8 @@ describe('Assistance', function () { ); checkModels([ OpenAIAssistant.DEFAULT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, + LONGER_CONTEXT_MODEL_FOR_TEST, + LONGER_CONTEXT_MODEL_FOR_TEST, ]); }); @@ -310,8 +323,8 @@ describe('Assistance', function () { const result = await checkSendForCompletion(); checkModels([ OpenAIAssistant.DEFAULT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, - OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL, + LONGER_CONTEXT_MODEL_FOR_TEST, + LONGER_CONTEXT_MODEL_FOR_TEST, ]); assert.deepEqual(result.suggestedActions, [ ["ModifyColumn", table1Id, colId, {formula: "123"}]