(core) Porting the AI evaluation script

Summary:
Porting script that run an evaluation against our formula dataset.

To test you need an openai key (see here: https://platform.openai.com/)
or hugging face (it should work as well), then checkout the branch and run

`OPENAI_API_KEY=<my_openai_api_key> node core/test/formula-dataset/runCompletion.js`

Test Plan:
Needs manually testing: so far there is no plan to make it part of CI.

The current score is somewhere around 34 successful prompts over a total of 47.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: jarek

Differential Revision: https://phab.getgrist.com/D3816
This commit is contained in:
Cyprien P
2023-03-15 09:52:17 +01:00
parent 2b2e19c5b5
commit 1ff93f89c2
8 changed files with 377 additions and 37 deletions

View File

@@ -1,3 +1,4 @@
import {Role} from 'app/common/roles';
import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DummyAuthorizer} from 'app/server/lib/Authorizer';
@@ -82,8 +83,8 @@ export function createDocTools(options: {persistAcrossCases?: boolean,
const systemSession = makeExceptionalDocSession('system');
return {
/** create a fake session for use when applying user actions to a document */
createFakeSession(): DocSession {
return {client: null, authorizer: new DummyAuthorizer('editors', 'doc')} as any as DocSession;
createFakeSession(role: Role = 'editors'): DocSession {
return {client: null, authorizer: new DummyAuthorizer(role, 'doc')} as any as DocSession;
},
/** create a throw-away, empty document for testing purposes */