From 5cdc7b2ea45dbac499b9ba41eb9d1064cbf2c603 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Fri, 7 Jan 2022 12:06:04 -0500 Subject: [PATCH] (core) freshen core README; support python3 in grist-core docker image Summary: This updates the grist-core README to list specific features of Grist, to make it easier for a casual visitor to get a sense of its scope. Adds links to some new resources (reviews, templates, grist v airtable post) that could also help. Adds python3 to docker image so that templates work without fuss. Test Plan: existing tests should pass Reviewers: georgegevoian Reviewed By: georgegevoian Subscribers: dsagal, anaisconce Differential Revision: https://phab.getgrist.com/D3204 --- Dockerfile | 42 ++++++--- LICENSE.txt | 2 +- NOTICE.txt | 2 +- README.md | 172 ++++++++++++++++++++++++---------- app/server/lib/NSandbox.ts | 16 ++-- buildtools/prepare_python.sh | 16 ++-- buildtools/prepare_python2.sh | 14 +++ buildtools/prepare_python3.sh | 9 +- package.json | 2 + 9 files changed, 195 insertions(+), 80 deletions(-) create mode 100755 buildtools/prepare_python2.sh diff --git a/Dockerfile b/Dockerfile index 0bab88d8..c005da0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ################################################################################ -## Build stage +## Javascript build stage ################################################################################ FROM node:14-buster as builder @@ -18,12 +18,21 @@ ADD static static ADD test/tsconfig.json test/tsconfig.json RUN yarn run build:prod +################################################################################ +## Python collection stage +################################################################################ + +# Fetch python3.9 and python2.7 +FROM python:3.9-slim-buster as collector + # Install all python dependencies. ADD sandbox/requirements.txt requirements.txt +ADD sandbox/requirements3.txt requirements3.txt RUN \ apt update && \ - apt install -y python-pip && \ - pip install -r requirements.txt + apt install -y --no-install-recommends python2 python-pip python-setuptools && \ + pip2 install -r requirements.txt && \ + pip3 install -r requirements3.txt ################################################################################ ## Run-time stage @@ -32,16 +41,29 @@ RUN \ # Now, start preparing final image. FROM node:14-buster-slim +# Install libexpat1, libsqlite3-0 for python3 library binary dependencies. +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends libexpat1 libsqlite3-0 && \ + rm -rf /var/lib/apt/lists/* + +# Keep all storage user may want to persist in a distinct directory +RUN mkdir -p /persist/docs + # Copy node files. COPY --from=builder /node_modules node_modules COPY --from=builder /_build _build COPY --from=builder /static static -# Copy python files. TODO: package python3.9 also in grist-core. -COPY --from=builder /usr/bin/python2.7 /usr/bin/python2.7 -COPY --from=builder /usr/lib/python2.7 /usr/lib/python2.7 -COPY --from=builder /usr/local/lib/python2.7 /usr/local/lib/python2.7 -RUN ln -s /usr/bin/python2.7 /usr/bin/python +# Copy python files. +COPY --from=collector /usr/bin/python2.7 /usr/bin/python2.7 +COPY --from=collector /usr/lib/python2.7 /usr/lib/python2.7 +COPY --from=collector /usr/local/lib/python2.7 /usr/local/lib/python2.7 +COPY --from=collector /usr/local/bin/python3.9 /usr/bin/python3.9 +COPY --from=collector /usr/local/lib/python3.9 /usr/local/lib/python3.9 +COPY --from=collector /usr/local/lib/libpython3.9.* /usr/local/lib/ +# Set default to python3 +RUN ln -s /usr/bin/python3.9 /usr/bin/python && ldconfig # Add files needed for running server. ADD package.json package.json @@ -50,13 +72,11 @@ ADD bower_components bower_components ADD sandbox sandbox ADD plugins plugins -# Keep all storage user may want to persist in a distinct directory -RUN mkdir -p /persist/docs - # Set some default environment variables to give a setup that works out of the box when # started as: # docker run -p 8484:8484 -it # Variables will need to be overridden for other setups. +ENV PYTHON_VERSION_ON_CREATION=3 ENV GRIST_ORG_IN_PATH=true ENV GRIST_HOST=0.0.0.0 ENV GRIST_SINGLE_PORT=true diff --git a/LICENSE.txt b/LICENSE.txt index db93ea0d..40b8ed33 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2014-2020 Grist Labs Inc. + Copyright 2014-2022 Grist Labs Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/NOTICE.txt b/NOTICE.txt index aeedf45b..0ca65d78 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Grist Software -Copyright 2014-2021 Grist Labs Inc. +Copyright 2014-2022 Grist Labs Inc. This product includes software developed at Grist Labs Inc. (https://www.getgrist.com/). diff --git a/README.md b/README.md index 5eeac8eb..2baaacc8 100644 --- a/README.md +++ b/README.md @@ -3,92 +3,162 @@ Grist is a modern relational spreadsheet. It combines the flexibility of a spreadsheet with the robustness of a database to organize your data and make you more productive. -> :warning: This repository is in a pre-release state. Its release will be announced when it has -all the planned components, and a solid independent build and test set-up. Currently, stand-alone -server functionality is present, along with a single-user web client. +## Features + +(By popular request: we have a specific write-up of [Grist vs Airtable](https://www.getgrist.com/blog/grist-v-airtable/) that may be helpful). +Grist is a hybrid database/spreadsheet, meaning that: + + - Columns work like they do in databases. They are named, and hold one kind of data. + - Columns can be filled by formula, spreadsheet-style, with automatic updates when referenced cells change. + +Here are some specific feature highlights of Grist: + + * Python formulas. + - Full [Python syntax is supported](https://support.getgrist.com/formulas/#python), and the standard library. + - Many [Excel functions](https://support.getgrist.com/functions/) also available. + * A portable, self-contained format. + - Based on SQLite, the most widely deployed database engine. + - Any tool that can read SQLite can read numeric and text data from a Grist file. + - Great format for [backups](https://support.getgrist.com/exports/#backing-up-an-entire-document) that you can be confident you can restore in full. + - Great format for moving between different hosts. + * Convenient editing and formatting features. + - Choices and [choice lists](https://support.getgrist.com/col-types/#choice-list-columns), for adding colorful tags to records without fuss. + - [References](https://support.getgrist.com/col-refs/#creating-a-new-reference-list-column) and reference lists, for cross-referencing records in other tables. + - [Attachments](https://support.getgrist.com/col-types/#attachment-columns), to include media or document files in records. + - Dates and times, toggles, and special numerics such as currency all have specialized editors and formatting options. + * Great for dashboards, visualizations, and data entry. + - [Charts](https://support.getgrist.com/widget-chart/) for visualization. + - [Summary tables](https://support.getgrist.com/summary-tables/) for summing and counting across groups. + - [Widget linking](https://support.getgrist.com/linking-widgets/) streamlines filtering and editing data. + Grist has a unique approach to visualization, where you can lay out and link distinct widgets to show together, + without cramming mixed material into a table. + - The [Filter bar](https://support.getgrist.com/search-sort-filter/#filter-buttons) is great for quick slicing and dicing. + * [Incremental imports](https://support.getgrist.com/imports/#updating-existing-records). + - So you can import a CSV of the last three months activity from your bank... + - ... and import new activity a month later without fuss or duplicates. + * Integrations. + - A [REST API](https://support.getgrist.com/api/), [Zapier actions/triggers](https://support.getgrist.com/integrators/#integrations-via-zapier), and support from similar [integrators](https://support.getgrist.com/integrators/). + - Import/export to Google drive, Excel format, CSV. + - Can link data with custom widgets hosted externally. + * [Many templates](https://templates.getgrist.com/) to get you started, from investment research to organizing treasure hunts. + * Access control options. + - (You'll need SSO logins set up to make use of these options) + - Share [individual documents](https://support.getgrist.com/sharing/), or workspaces, or [team sites](https://support.getgrist.com/team-sharing/). + - Control access to [individual rows, columns, and tables](https://support.getgrist.com/access-rules/). + - Control access based on cell values and user attributes. + * Can be self-maintained. + - Useful for intranet operation and specific compliance requirements. -This repository, [grist-core](https://github.com/gristlabs/grist-core), is maintained by Grist -Labs. Our flagship product, available at [getgrist.com](https://www.getgrist.com), is built from the code you see -here, combined with business-specific software designed to scale it to many users, handle billing, -etc. - -If you are looking to use Grist in the cloud, head on over to [getgrist.com](https://www.getgrist.com). If you are curious about where Grist is going heading, see [our roadmap](https://github.com/gristlabs/grist-core/projects/1), drop a question in [our forum](https://community.getgrist.com), or browse [our extensive documentation](https://support.getgrist.com). +## Using Grist -## Opening and editing a Grist document locally +There are docker images set up for individual use, or (with some +configuration) for self-hosting. Grist Labs offers a hosted service +at [https://docs.getgrist.com](docs.getgrist.com). -The easiest way to use Grist locally on your computer is with [Docker](https://www.docker.com/get-started). -From a terminal, do: +To run Grist running on your computer with [Docker](https://www.docker.com/get-started), do: ```sh docker pull gristlabs/grist docker run -p 8484:8484 -it gristlabs/grist ``` -Then visit `http://localhost:8484` in your browser. You'll be able to create and edit documents, -and to import documents downloaded from the https://docs.getgrist.com host. You'll also be able -to use the Grist API. - -To preserve your work across docker runs, provide a directory to save it in: +Then visit `http://localhost:8484` in your browser. You'll be able to create, edit, import, +and export documents. To preserve your work across docker runs, share a directory as `/persist`: ```sh -docker pull gristlabs/grist docker run -p 8484:8484 -v $PWD/persist:/persist -it gristlabs/grist ``` -## Building from source +Get templates at https://templates.getgrist.com/ for payroll, +inventory management, invoicing, D&D encounter tracking, and a lot +more, or use any document you've created on +[https://docs.getgrist.com](docs.getgrist.com). -Here are the steps needed: +If you need to change the port Grist runs on, set a `PORT` variable, don't just change the +port mapping: -```sh -yarn install -yarn run build:prod -yarn run install:python -yarn start -# grist client available at http://localhost:8484 -# grist api available at http://localhost:8484/api/ ``` +docker run --env PORT=9999 -p 9999:9999 -v $PWD/persist:/persist -it gristlabs/grist +``` + +## Building from source + +To build Grist from source, follow these steps: -Then you can use the Grist client, or the API. You can view and edit Grist documents -throught the client and the API. All imported/created documents will appear in the `docs` -subdirectory. You cannot (yet) edit Grist documents in place on your file system. + yarn install + yarn run build:prod + yarn run install:python + yarn start + # Grist will be available at http://localhost:8484/ -Grist does not have a login system built in. To activate one, you can configure Grist -to talk to an identity provider such as Auth0 using +## Logins + +Like git, Grist has features to track document revision history. So for full operation, +Grist expects to know who the user modifying a document is. Until it does, it operates +in a limited anonymous mode. To get you going, the docker image is configured so that +when you click on the "sign in" button Grist will attribute your work to `you@example.com`. +Change this by setting `GRIST_DEFAULT_EMAIL`: + +``` +docker run --env GRIST_DEFAULT_EMAIL=my@email -p 8484:8484 -v $PWD/persist:/persist -it gristlabs/grist +``` + +You can change your name in `Profile Settings` in +the [User Menu](https://support.getgrist.com/glossary/#user-menu). +For multi-user operation, and/or if you wish to access Grist across the +public internet, you'll want to connect it to your own single sign-in service [SAML](https://github.com/gristlabs/grist-core/blob/main/app/server/lib/SamlConfig.ts). -For running on your own computer, this isn't necessary, but it is important if you are -self-hosting Grist for use by a team. +Grist has been tested with [Authentik](https://goauthentik.io/) and [Auth0](https://auth0.com/). + +## Why free and open source software -## Why Open Source? +This repository, [grist-core](https://github.com/gristlabs/grist-core), is maintained by Grist +Labs. Our flagship product available at [getgrist.com](https://www.getgrist.com) is built from the code you see +here, combined with business-specific software designed to scale it to many users, handle billing, +etc. + +Grist Labs is an open-core company. We offer Grist hosting as a +service, with free and paid plans. We intend to also develop and sell +features related to Grist using a proprietary license, targeted at the +needs of enterprises with large self-managed installations. We see +data portability and autonomy as a key value Grist can bring to our +users, and `grist-core` as an essential means to deliver that. We are +committed to maintaining and improving the `grist-core` codebase, and +to be thoughtful about how proprietary offerings impact data portability +and autonomy. By opening its source code and offering an [OSI](https://opensource.org/)-approved free license, Grist benefits its users: -- **Open Source Community.** An active community is the main draw of open-source projects. Anyone - can examine source code, and contribute bug fixes or even new features. This is a big deal for a - general-purpose spreadsheet-like product, where there is a long tail of features vital to - someone somewhere. -- **Increased Trust.** Because anyone can examine the source code, “security by obscurity” is not - an option. Vulnerabilities in the code can be found by others and reported before they can cause +- **Developer community.** The freedom to examine source code, make bug fixes, and develop + new features is a big deal for a general-purpose spreadsheet-like product, where there is a + very long tail of features vital to someone somewhere. +- **Increased trust.** Because anyone can examine the source code, “security by obscurity” is not + an option. Vulnerabilities in the code can be found by others and reported before they cause damage. -- **Independence.** The published source code—and the product built from it—are available to you - regardless of the fortunes of the Grist Labs business. Whatever happens to us, this repo or its - forks can live on, so that you can continue to work on your data in Grist. -- **Price Flexibility.** You can build Grist from source and use it for yourself all you want - without paying us a cent. While you can’t go wrong with our fully set-up and supported online - service, some organizations may choose the do-it-yourself route and pay for their own server and - maintenance, rather than a per-user price. DIY users are often the ones to develop new features, - and can contribute them back to benefit all users of Grist. +- **Independence.** Grist is available to you regardless of the fortunes of the Grist Labs business, + since it is open source and can be self-hosted. Using our hosted solution is convenient, but you + are not locked in. +- **Price flexibility.** If you are low on funds but have time to invest, self-hosting is a great + option to have. And DIY users may have the technical savvy and motivation to delve in and make improvements, + which can benefit all users of Grist. - **Extensibility.** For developers, having the source open makes it easier to build extensions (such as the experimental [Custom Widget](https://support.getgrist.com/widget-custom/)). You can more easily include Grist in your pipeline. And if a feature is missing, you can just take the source code and - build on top of it! + build on top of it. + +## Reviews + + * [Grist on ProductHunt](https://www.producthunt.com/posts/grist-2) + * [Grist on AppSumo](https://appsumo.com/products/grist/) (life-time deal is sold out) + * [Capterra](https://www.capterra.com/p/232821/Grist/#reviews), [G2](https://www.g2.com/products/grist/reviews), [TrustRadius](https://www.trustradius.com/products/grist/reviews) -# License +## License This repository, `grist-core`, is released under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0), which is an diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts index d0f3fc53..5ccc5a02 100644 --- a/app/server/lib/NSandbox.ts +++ b/app/server/lib/NSandbox.ts @@ -401,7 +401,7 @@ export class NSandboxCreator implements ISandboxCreator { ...options.logMeta}, logTimes: options.logTimes, command: this._command, - preferredPythonVersion: this._preferredPythonVersion, + preferredPythonVersion: this._preferredPythonVersion || options.preferredPythonVersion, useGristEntrypoint: true, importDir: options.importMount, }; @@ -489,7 +489,7 @@ function unsandboxed(options: ISandboxOptions): SandboxProcess { if (!options.minimalPipeMode) { spawnOptions.stdio.push('pipe', 'pipe'); } - const command = findPython(options.command); + const command = findPython(options.command, options.preferredPythonVersion); const child = spawn(command, pythonArgs, {cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions}); return {child, control: new DirectProcessControl(child, options.logMeta)}; @@ -631,7 +631,7 @@ function macSandboxExec(options: ISandboxOptions): SandboxProcess { ...getInsertedEnv(options), ...getWrappingEnv(options), }; - const command = findPython(options.command); + const command = findPython(options.command, options.preferredPythonVersion); const realPath = fs.realpathSync(command); log.rawDebug("macSandboxExec found a python", {...options.logMeta, command: realPath}); @@ -805,13 +805,15 @@ const FAKETIME = '2020-01-01 00:00:00'; /** * Find a plausible version of python to run, if none provided. + * The preferred version is only used if command is not specified. */ -function findPython(command?: string) { +function findPython(command: string|undefined, preferredVersion?: string) { if (command) { return command; } // No command specified. In this case, grist-core looks for a "venv" // virtualenv; a python3 virtualenv would be in "sandbox_venv3". // TODO: rationalize this, it is a product of haphazard growth. - for (const venv of ['sandbox_venv3', 'venv']) { + const prefs = preferredVersion === '2' ? ['venv', 'sandbox_venv3'] : ['sandbox_venv3', 'venv']; + for (const venv of prefs) { const pythonPath = path.join(process.cwd(), venv, 'bin', 'python'); if (fs.existsSync(pythonPath)) { command = pythonPath; @@ -820,7 +822,9 @@ function findPython(command?: string) { } // Fall back on system python. if (!command) { - command = which.sync('python'); + command = which.sync(preferredVersion === '2' ? 'python2' : 'python3', {nothrow: true}) + || which.sync(preferredVersion === '2' ? 'python2.7' : 'python3.9', {nothrow: true}) + || which.sync('python'); } return command; } diff --git a/buildtools/prepare_python.sh b/buildtools/prepare_python.sh index 77a3d73c..05b793d2 100755 --- a/buildtools/prepare_python.sh +++ b/buildtools/prepare_python.sh @@ -2,10 +2,14 @@ set -e -if [ ! -e venv ]; then - virtualenv -ppython2.7 venv +echo "Use Python3 if available and recent enough, otherwise Python2" +if python3 -c 'import sys; assert sys.version_info >= (3,9)' 2> /dev/null; then + # Default to python3 if recent enough. + buildtools/prepare_python3.sh + # Make sure python2 isn't around. + rm -rf venv +else + buildtools/prepare_python2.sh + # Make sure python3 isn't around. + rm -rf sandbox_venv3 fi - -. venv/bin/activate - -pip install --no-deps -r sandbox/requirements.txt diff --git a/buildtools/prepare_python2.sh b/buildtools/prepare_python2.sh new file mode 100755 index 00000000..97e6f7a8 --- /dev/null +++ b/buildtools/prepare_python2.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +echo "Making Python2 sandbox" +if [ ! -e venv ]; then + virtualenv -ppython2.7 venv +fi + +. venv/bin/activate + +echo "Updating Python2 packages" +pip install --no-deps -r sandbox/requirements.txt +echo "Python2 packages ready in venv" diff --git a/buildtools/prepare_python3.sh b/buildtools/prepare_python3.sh index 5e9aee44..8df3a3a9 100755 --- a/buildtools/prepare_python3.sh +++ b/buildtools/prepare_python3.sh @@ -2,10 +2,11 @@ set -e +echo "Making Python3 sandbox" if [ ! -e sandbox_venv3 ]; then - virtualenv -ppython3 sandbox_venv3 + python3 -m venv sandbox_venv3 fi -. sandbox_venv3/bin/activate - -pip install --no-deps -r sandbox/requirements3.txt +echo "Updating Python3 packages" +sandbox_venv3/bin/pip install --no-deps -r sandbox/requirements3.txt +echo "Python3 packages ready in sandbox_venv3" diff --git a/package.json b/package.json index 256eefc9..6e15a9e3 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "scripts": { "start": "tsc --build -w --preserveWatchOutput & catw app/client/*.css app/client/*/*.css -o static/bundle.css -v & webpack --config buildtools/webpack.config.js --mode development --watch --hide-modules & NODE_PATH=_build:_build/stubs nodemon --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js & wait", "install:python": "buildtools/prepare_python.sh", + "install:python2": "buildtools/prepare_python2.sh", + "install:python3": "buildtools/prepare_python3.sh", "build:prod": "tsc --build && webpack --config buildtools/webpack.config.js --mode production && webpack --config buildtools/webpack.check.js --mode production && cat app/client/*.css app/client/*/*.css > static/bundle.css", "start:prod": "NODE_PATH=_build:_build/stubs node _build/stubs/app/server/server.js", "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support NODE_PATH=_build:_build/stubs mocha _build/test/nbrowser/*.js",