Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

switch default LLM from a model that is going away (#1202)

Browse Source 
* switch default LLM from a model that is going away

If an api key is provided, Grist can use an LLM as an assistant
for writing formulas. The LLM can be self-hosted or an external
service. The default external service is OpenAI. This commit
freshens the default model used, if this feature is enabled,
since the existing one is going away. Benchmarking suggests the
results are generally better, though not dramatically so.

The feature of falling back on a longer context model is no longer
as important, but is retained since it could be useful for self-hosters.

* update long context model tests
This commit is contained in:
Paul Fitzpatrick 2024-09-13 14:56:42 -04:00 committed by GitHub
parent 43f7fbeff4
commit a45a7f9fda
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 28 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/server/lib/Assistance.ts
View File

@ -148,8 +148,8 @@ class RetryableError extends Error {
* An optional ASSISTANT_MAX_TOKENS can be specified.
*/
export class OpenAIAssistant implements Assistant {
public static DEFAULT_MODEL = "gpt-3.5-turbo-0613";
public static DEFAULT_LONGER_CONTEXT_MODEL = "gpt-3.5-turbo-16k-0613";
public static DEFAULT_MODEL = "gpt-4o-2024-08-06";
public static DEFAULT_LONGER_CONTEXT_MODEL = "";
private _apiKey?: string;
private _model?: string;

39
test/server/lib/Assistance.ts
View File

@ -1,11 +1,12 @@
import {createDocTools} from "test/server/docTools";
import {AssistanceState} from 'app/common/AssistancePrompts';
import {ActiveDoc} from "app/server/lib/ActiveDoc";
import {DEPS, OpenAIAssistant, sendForCompletion} from "app/server/lib/Assistance";
import {DEPS, OpenAIAssistant, sendForCompletion} from 'app/server/lib/Assistance';
import {DocSession} from 'app/server/lib/DocSession';
import {assert} from 'chai';
import * as sinon from 'sinon';
import {Response} from 'node-fetch';
import {DocSession} from "app/server/lib/DocSession";
import {AssistanceState} from "app/common/AssistancePrompts";
import * as sinon from 'sinon';
import {createDocTools} from 'test/server/docTools';
import {EnvironmentSnapshot} from 'test/server/testUtils';
// For some reason, assert.isRejected is not getting defined,
// though test/chai-as-promised.js should be taking care of this.
@ -14,6 +15,12 @@ const chai = require('chai');
const chaiAsPromised = require('chai-as-promised');
chai.use(chaiAsPromised);
/**
* We no longer use a longer context model by default, but we still
* test this configuration.
*/
const LONGER_CONTEXT_MODEL_FOR_TEST = "fake";
describe('Assistance', function () {
this.timeout(10000);
@ -22,8 +29,11 @@ describe('Assistance', function () {
const table2Id = "Table2";
let session: DocSession;
let doc: ActiveDoc;
let oldEnv: EnvironmentSnapshot;
before(async () => {
oldEnv = new EnvironmentSnapshot();
process.env.OPENAI_API_KEY = "fake";
process.env.ASSISTANT_LONGER_CONTEXT_MODEL = LONGER_CONTEXT_MODEL_FOR_TEST;
session = docTools.createFakeSession();
doc = await docTools.createDoc('test.grist');
await doc.applyUserActions(session, [
@ -31,6 +41,9 @@ describe('Assistance', function () {
["AddTable", table2Id, [{id: "A"}, {id: "B"}, {id: "C"}]],
]);
});
after(async function () {
oldEnv.restore();
});
const colId = "C";
const userMessageContent = "Sum of A and B";
@ -204,8 +217,8 @@ describe('Assistance', function () {
);
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
});
@ -254,8 +267,8 @@ describe('Assistance', function () {
);
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
});
@ -278,8 +291,8 @@ describe('Assistance', function () {
);
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
});
@ -310,8 +323,8 @@ describe('Assistance', function () {
const result = await checkSendForCompletion();
checkModels([
OpenAIAssistant.DEFAULT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
OpenAIAssistant.DEFAULT_LONGER_CONTEXT_MODEL,
LONGER_CONTEXT_MODEL_FOR_TEST,
LONGER_CONTEXT_MODEL_FOR_TEST,
]);
assert.deepEqual(result.suggestedActions, [
["ModifyColumn", table1Id, colId, {formula: "123"}]