Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

switch default LLM from a model that is going away (#1202)

Browse Source 
* switch default LLM from a model that is going away

If an api key is provided, Grist can use an LLM as an assistant
for writing formulas. The LLM can be self-hosted or an external
service. The default external service is OpenAI. This commit
freshens the default model used, if this feature is enabled,
since the existing one is going away. Benchmarking suggests the
results are generally better, though not dramatically so.

The feature of falling back on a longer context model is no longer
as important, but is retained since it could be useful for self-hosters.

* update long context model tests
This commit is contained in:
Paul Fitzpatrick
2024-09-13 14:56:42 -04:00
committed by GitHub
parent 43f7fbeff4
commit a45a7f9fda
2 changed files with 28 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
test/server/lib/Assistance.ts
View File

@@ -1,11 +1,12 @@
import {createDocTools} from "test/server/docTools";
import {AssistanceState} from 'app/common/AssistancePrompts';
import {ActiveDoc} from "app/server/lib/ActiveDoc";
import {DEPS, OpenAIAssistant, sendForCompletion} from "app/server/lib/Assistance";
import {DEPS, OpenAIAssistant, sendForCompletion} from 'app/server/lib/Assistance';
import {DocSession} from 'app/server/lib/DocSession';
import {assert} from 'chai';
import * as sinon from 'sinon';
import {Response} from 'node-fetch';
import {DocSession} from "app/server/lib/DocSession";
import {AssistanceState} from "app/common/AssistancePrompts";
import * as sinon from 'sinon';
import {createDocTools} from 'test/server/docTools';
import {EnvironmentSnapshot} from 'test/server/testUtils';
// For some reason, assert.isRejected is not getting defined,
// though test/chai-as-promised.js should be taking care of this.
@@ -14,6 +15,12 @@ const chai = require('chai');
const chaiAsPromised = require('chai-as-promised');
chai.use(chaiAsPromised);
/**
* We no longer use a longer context model by default, but we still
* test this configuration.
*/
const LONGER_CONTEXT_MODEL_FOR_TEST = "fake";
describe('Assistance', function () {
this.timeout(10000);
@@ -22,8 +29,11 @@ describe('Assistance', function () {
const table2Id = "Table2";
let session: DocSession;
let doc: ActiveDoc;
let oldEnv: EnvironmentSnapshot;
before(async () => {
oldEnv = new EnvironmentSnapshot();
process.env.OPENAI_API_KEY = "fake";
process.env.ASSISTANT_LONGER_CONTEXT_MODEL = LONGER_CONTEXT_MODEL_FOR_TEST;
session = docTools.createFakeSession();
doc = await docTools.createDoc('test.grist');
await doc.applyUserActions(session, [
@@ -31,6 +41,9 @@ describe('Assistance', function () {
["AddTable", table2Id, [{id: "A"}, {id: "B"}, {id: "C"}]],
]);
});
after(async function () {
oldEnv.restore();
});
const colId = "C";
const userMessageContent = "Sum of A and B";
@@ -204,8 +217,8 @@ describe('Assistance', function () {
);
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
});
@@ -254,8 +267,8 @@ describe('Assistance', function () {
);
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
});
@@ -278,8 +291,8 @@ describe('Assistance', function () {
);
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
});
@@ -310,8 +323,8 @@ describe('Assistance', function () {
const result = await checkSendForCompletion();
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
assert.deepEqual(result.suggestedActions, [
["ModifyColumn", table1Id, colId, {formula: "123"}]