Compare commits

..

28 Commits
3.0.1 ... 3.1.0

Author SHA1 Message Date
Athou
0b16b6bb86 release 3.1.0 2023-05-01 18:25:16 +02:00
Athou
6a8f7f0a40 add release script 2023-05-01 18:23:35 +02:00
Athou
42ca0967b6 create release on tag 2023-05-01 17:39:01 +02:00
Athou
deb29f0e88 fix metrics page 2023-05-01 17:07:17 +02:00
Athou
714af986b0 readme update 2023-05-01 17:05:38 +02:00
Athou
4ff26366a5 there's no need to update disabledUntil here anymore because findNextUpdatableFeeds will always be called when the queue is empty 2023-05-01 10:04:43 +02:00
Athou
9c628a8f53 make each step of feed fetching return its own model 2023-05-01 09:58:19 +02:00
Athou
4a40f2b8f7 no need to log if we're not sending any notification 2023-04-30 23:05:48 +02:00
Athou
9a2dda626c changelog update 2023-04-30 23:05:48 +02:00
Athou
a9ff491da0 hide request log in production 2023-04-30 16:21:38 +02:00
Athou
5c5a7d20de in production, no need to see warnings 2023-04-30 16:03:47 +02:00
Athou
05ae4eb529 replace homemade threadpool framework with rxjava 2023-04-30 15:34:32 +02:00
Athou
15f93b198c remove warning 2023-04-29 09:20:34 +02:00
Athou
0a99dacb6b use urlcanon instead of crawler4j because we only used it for url canonization 2023-04-29 09:20:15 +02:00
Athou
00f6c04611 various dependency updates 2023-04-29 09:17:59 +02:00
Athou
d9b899b53f prevent entries from having the hover background color when clicked on mobile 2023-04-28 19:44:26 +02:00
Athou
d96f8da8fd remove deprecation warnings (already done in config.yml.example) 2023-04-28 19:44:26 +02:00
Athou
ababcf7850 remove unnecessary "subscriptions" field on Feed
hopefully removes the error that happens sometimes:
Illegal attempt to associate a collection with two open sessions. Collection : [com.commafeed.backend.model.Feed.subscriptions]
2023-04-28 19:44:26 +02:00
Athou
f23bfaf694 use a different color for hover than read and unread backgrounds 2023-04-28 19:44:26 +02:00
Athou
cac05dee0b store view mode in localStorage (#1051) 2023-04-27 14:42:55 +02:00
Athou
155c93d371 reorder Dockerfile to put changing layers last 2023-04-27 09:38:23 +02:00
Athou
9a61ee7530 create a 'master' docker tag for the latest master version 2023-04-27 09:38:23 +02:00
Athou
4bea1c5e5c restore hover effect from commafeed v2 2023-04-27 07:57:09 +02:00
Athou
9ccc26b0b0 tweak compact layout a little bit more 2023-04-27 07:30:43 +02:00
Athou
5cd3787d6f add i18n placeholders for new label 2023-04-26 23:37:39 +02:00
Athou
807b1f62a1 add an even more compact entry layout 2023-04-26 22:50:43 +02:00
Athou
c15db54d5a bump versions 2023-04-26 09:43:10 +02:00
Athou
aa7b078121 readme update 2023-04-25 15:34:20 +02:00
76 changed files with 19666 additions and 19666 deletions

View File

@@ -10,7 +10,8 @@ jobs:
java: [ "8", "11", "17" ]
steps:
- uses: actions/checkout@v3
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
@@ -32,26 +33,49 @@ jobs:
- name: Build with Maven
run: mvn --batch-mode --update-snapshots verify
- uses: actions/upload-artifact@v3
- name: Upload JAR
uses: actions/upload-artifact@v3
if: ${{ matrix.java == '8' }}
with:
name: commafeed.jar
path: commafeed-server/target/commafeed.jar
# Docker
- name: Create release
uses: softprops/action-gh-release@v1
if: ${{ matrix.java == '8' && github.ref_type == 'tag' }}
with:
name: CommaFeed ${{ github.ref_name }}
body: See changelog at https://github.com/Athou/commafeed/blob/master/CHANGELOG.md
draft: false
prerelease: false
files: |
commafeed-server/target/commafeed.jar
commafeed-server/config.yml.example
# Docker
- name: Login to Container Registry
if: ${{ github.ref_type == 'tag' && matrix.java == '8' }}
uses: docker/login-action@v2
if: ${{ matrix.java == '8' }}
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Docker build and push
- name: Docker build and push tag
uses: docker/build-push-action@v4
if: ${{ matrix.java == '8' && github.ref_type == 'tag' }}
with:
context: .
push: true
platforms: linux/amd64,linux/arm/v7
push: ${{ github.ref_type == 'tag' && matrix.java == '8' }}
tags: |
athou/commafeed:latest
athou/commafeed:${{ github.ref_name }}
- name: Docker build and push master
uses: docker/build-push-action@v4
if: ${{ matrix.java == '8' && github.ref_name == 'master' }}
with:
context: .
push: true
platforms: linux/amd64,linux/arm/v7
tags: athou/commafeed:master

View File

@@ -1,80 +0,0 @@
v 3.0.0
- complete overhaul of the UI
- backend and frontend are now in separate maven modules
- no changes to the api or the database
v 2.6.0
- add support for media content as a backup for missing content (useful for youtube feeds)
- correctly follow http error code 308 redirects
- fixed a bug that prevented users from deleting their account
- fixed a bug that made commafeed store entry contents multiple times
- fixed a bug that prevented the app to be used as an installed app on mobile devices if the context path of commafeed was not "/"
- fixed a bug that prevented entries from being "marked as read older than xxx" for a feed that was just added
- removed support for google+ and readability as those services no longer exist
- removed support for deploying on openshift
- removed alphabetical sorting of entries because of really poor performance (title cannot be indexed)
- improve performance for instances with the heavy load setting enabled by preventing CommaFeed from fetching feeds from users that did not log in for a long time
- various dependencies upgrades (notably dropwizard from 1.3 to 2.1)
- add support for mariadb
- add support for java17+ runtime
- various security improvements
v 2.5.0
- unread count is now displayed in a favicon badge when supported
- the user agent string for the bot fetching feeds is now configurable
- feed parsing performance improvements
- support for java9+ runtime
- can now properly start from an empty postgresql database
v 2.4.0
- users were not able to change password or delete account
- fix api key generation
- feed entries can now be sorted alphabetically
- fix facebook sharing
- fix layout on iOS
- postgresql driver update (fix for postgres 9.6)
- various internationalization fixes
- security fixes
v 2.3.0
- dropwizard upgrade 0.9.1
- feed enclosures are hidden if they already displayed in the content
- fix youtube favicons
- various internationalization fixes
v 2.2.0
- fix youtube and instagram favicon fetching
- mark as read filter was lost when a feed was rearranged with drag&drop
- feed entry categories are now displayed if available
- various performance and dependencies upgrades
- java8 is now required
v 2.1.0
- dropwizard upgrade to 0.8.0
- you have to remove the "app.contextPath" setting from your yml file, you can optionally use server.applicationContextPath instead
- new setting app.maxFeedCapacity for deleting old entries
- ability to set filtering expressions for subscriptions to automatically mark new entries as read based on title, content, author or url.
- ability to use !keyword or -keyword to exclude a keyword from a search query
- facebook feeds now show user favicon instead of facebook favicon
- new dark theme 'nightsky'
v 2.0.3
- internet explorer ajax cache workaround
- categories are now deletable again
- openshift support is back
- youtube feeds now show user favicon instead of youtube favicon
v 2.0.2
- api using the api key is now working again
- context path is now configurable in config.yml (see app.contextPath in config.yml.example)
- fix login on firefox when fields are autofilled by the browser
- fix scrolling of subscriptions list on mobile
- user is now logged in after registration
- fix link to documentation on home page and about page
- fields autocomplete is disabled on the profile page
- users are able to delete their account again
- chinese and malaysian translation files are now correctly loaded
- software version in user-agent when fetching feeds is no longer hardcoded
- admin settings page is now read only, settings are configured in config.yml
- added link to metrics on the admin settings page
- Rome (rss library) upgrade to 1.5.0
v 2.0.1
- the redis pool no longer throws an exception when it is unable to aquire a new connection
v2.0.0
- The backend has been completely rewritten using Dropwizard instead of TomEE, resulting in a lot less memory consumption and better overall performances.
See the README on how to build CommaFeed from now on.
- CommaFeed should no longer fetch the same feed multiple times in a row
- Users can use their username or email to log in

139
CHANGELOG.md Normal file
View File

@@ -0,0 +1,139 @@
Changelog
=========
3.1.0
-----
- add an even more compact layout
- restore hover effect from commafeed 2.x
- view mode (compact, expanded, ...) is now stored on the device so you can have a different view mode on desktop and
mobile
- fix for the "Illegal attempt to associate a collection with two open sessions." error
- feed fetching workflow is now orchestrated with rxjava, removing a lot of code
3.0.1
-----
- allow env variable substitution in config.yml
- e.g. having a custom config.yml file with `app.session.path=${SOME_ENV_VAR}` will substitute `SOME_ENV_VAR` with
its value
- allow env variable prefixed with `CF_` to override config.yml properties
- e.g. setting `CF_APP_ALLOWREGISTRATIONS=true` will set `app.allowRegistrations` to `true`
3.0.0
-----
- complete overhaul of the UI
- backend and frontend are now in separate maven modules
- no changes to the api or the database
- Docker images are now automatically built and available at https://hub.docker.com/r/athou/commafeed
2.6.0
-----
- add support for media content as a backup for missing content (useful for youtube feeds)
- correctly follow http error code 308 redirects
- fixed a bug that prevented users from deleting their account
- fixed a bug that made commafeed store entry contents multiple times
- fixed a bug that prevented the app to be used as an installed app on mobile devices if the context path of commafeed
was not "/"
- fixed a bug that prevented entries from being "marked as read older than xxx" for a feed that was just added
- removed support for google+ and readability as those services no longer exist
- removed support for deploying on openshift
- removed alphabetical sorting of entries because of really poor performance (title cannot be indexed)
- improve performance for instances with the heavy load setting enabled by preventing CommaFeed from fetching feeds from
users that did not log in for a long time
- various dependencies upgrades (notably dropwizard from 1.3 to 2.1)
- add support for mariadb
- add support for java17+ runtime
- various security improvements
2.5.0
-----
- unread count is now displayed in a favicon badge when supported
- the user agent string for the bot fetching feeds is now configurable
- feed parsing performance improvements
- support for java9+ runtime
- can now properly start from an empty postgresql database
2.4.0
-----
- users were not able to change password or delete account
- fix api key generation
- feed entries can now be sorted alphabetically
- fix facebook sharing
- fix layout on iOS
- postgresql driver update (fix for postgres 9.6)
- various internationalization fixes
- security fixes
2.3.0
-----
- dropwizard upgrade 0.9.1
- feed enclosures are hidden if they already displayed in the content
- fix youtube favicons
- various internationalization fixes
2.2.0
-----
- fix youtube and instagram favicon fetching
- mark as read filter was lost when a feed was rearranged with drag&drop
- feed entry categories are now displayed if available
- various performance and dependencies upgrades
- java8 is now required
2.1.0
-----
- dropwizard upgrade to 0.8.0
- you have to remove the "app.contextPath" setting from your yml file, you can optionally use
server.applicationContextPath instead
- new setting app.maxFeedCapacity for deleting old entries
- ability to set filtering expressions for subscriptions to automatically mark new entries as read based on title,
content, author or url.
- ability to use !keyword or -keyword to exclude a keyword from a search query
- facebook feeds now show user favicon instead of facebook favicon
- new dark theme 'nightsky'
2.0.3
-----
- internet explorer ajax cache workaround
- categories are now deletable again
- openshift support is back
- youtube feeds now show user favicon instead of youtube favicon
2.0.2
-----
- api using the api key is now working again
- context path is now configurable in config.yml (see app.contextPath in config.yml.example)
- fix login on firefox when fields are autofilled by the browser
- fix scrolling of subscriptions list on mobile
- user is now logged in after registration
- fix link to documentation on home page and about page
- fields autocomplete is disabled on the profile page
- users are able to delete their account again
- chinese and malaysian translation files are now correctly loaded
- software version in user-agent when fetching feeds is no longer hardcoded
- admin settings page is now read only, settings are configured in config.yml
- added link to metrics on the admin settings page
- Rome (rss library) upgrade to 1.5.0
2.0.1
-----
- the redis pool no longer throws an exception when it is unable to aquire a new connection
2.0.0
-----
- The backend has been completely rewritten using Dropwizard instead of TomEE, resulting in a lot less memory
consumption and better overall performances.
See the README on how to build CommaFeed from now on.
- CommaFeed should no longer fetch the same feed multiple times in a row
- Users can use their username or email to log in

View File

@@ -1,12 +1,12 @@
FROM eclipse-temurin:17-jre
EXPOSE 8082
RUN mkdir -p /commafeed/data
VOLUME /commafeed/data
ENV CF_SESSION_PATH=/commafeed/data/sessions
COPY commafeed-server/target/commafeed.jar .
COPY commafeed-server/config.yml.example config.yml
COPY commafeed-server/target/commafeed.jar .
EXPOSE 8082
CMD ["java", "-Djava.net.preferIPv4Stack=true", "-jar", "commafeed.jar", "server", "config.yml"]

View File

@@ -1,9 +1,20 @@
# CommaFeed
Sources for [CommaFeed.com](http://www.commafeed.com/).
Google Reader inspired self-hosted RSS reader, based on Dropwizard and React/TypeScript.
Google Reader inspired self-hosted RSS reader, based on Dropwizard and AngularJS.
CommaFeed is now considered feature-complete and is in maintenance mode.
![preview](https://user-images.githubusercontent.com/1256795/184886828-1973f148-58a9-4c6d-9587-ee5e5d3cc2cb.png)
## Features
- 4 different layouts
- Dark theme
- Fully responsive
- Keyboard shortcuts for almost everything
- Support for right-to-left feeds
- Translated in 25+ languages
- Supports thousands of users and millions of feeds
- OPML import/export
- REST API
## Related open-source projects
@@ -23,9 +34,8 @@ Docker images are built automatically and are available at https://hub.docker.co
### Download precompiled package
mkdir commafeed && cd commafeed
wget https://github.com/Athou/commafeed/releases/download/3.0.0/commafeed.jar
wget https://raw.githubusercontent.com/Athou/commafeed/3.0.0/commafeed-server/config.yml.example -O config.yml
vi config.yml
wget https://github.com/Athou/commafeed/releases/latest/download/commafeed.jar
wget https://github.com/Athou/commafeed/releases/latest/download/config.yml.example -O config.yml
java -Djava.net.preferIPv4Stack=true -jar commafeed.jar server config.yml
The server will listen on http://localhost:8082. The default
@@ -37,7 +47,6 @@ user is `admin` and the default password is `admin`.
cd commafeed
./mvnw clean package
cp commafeed-server/config.yml.example config.yml
vi config.yml
java -Djava.net.preferIPv4Stack=true -jar commafeed-server/target/commafeed.jar server config.yml
The server will listen on http://localhost:8082. The default
@@ -65,8 +74,6 @@ two-letters [ISO-639-1 language code](http://en.wikipedia.org/wiki/List_of_ISO_6
- Open `commafeed-server` in your preferred Java IDE.
- CommaFeed uses Lombok, you need the Lombok plugin for your IDE.
- If using Eclipse, Go to Window → Preferences → Maven → Annotation Processing and check "Automatically configure
JDT APT"
- Start `CommaFeedApplication.java` in debug mode with `server config.dev.yml` as arguments
### Frontend
@@ -74,8 +81,9 @@ two-letters [ISO-639-1 language code](http://en.wikipedia.org/wiki/List_of_ISO_6
- Open `commafeed-client` in your preferred JavaScript IDE.
- run `npm install`
- run `npm run dev`
- the frontend server is now running at http://localhost:8082 and is proxying REST requests to the backend running on
port 8083
The frontend server is now running at http://localhost:8082 and is proxying REST requests to the backend running on
port 8083
## Copyright and license

File diff suppressed because it is too large Load Diff

View File

@@ -1,80 +1,82 @@
{
"name": "commafeed-client",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite --host",
"dev:typescript": "tsc --watch",
"build": "npm run i18n:compile && tsc && vite build",
"preview": "vite preview",
"test": "vitest",
"test:ci": "vitest run",
"eslint": "eslint --ext=.js,.jsx,.ts,.tsx src",
"i18n": "npm run i18n:extract && npm run i18n:compile",
"i18n:extract": "lingui extract --clean",
"i18n:compile": "lingui compile --typescript",
"postinstall": "npm run i18n:compile"
},
"dependencies": {
"@emotion/react": "^11.10.5",
"@fontsource/open-sans": "^4.5.14",
"@lingui/core": "^3.17.0",
"@lingui/macro": "^3.17.0",
"@lingui/react": "^3.17.0",
"@mantine/core": "^5.10.3",
"@mantine/form": "^5.10.3",
"@mantine/hooks": "^5.10.3",
"@mantine/modals": "^5.10.3",
"@mantine/notifications": "^5.10.3",
"@mantine/spotlight": "^5.10.3",
"@reduxjs/toolkit": "^1.9.2",
"axios": "^1.3.2",
"dayjs": "^1.11.7",
"interweave": "^13.0.0",
"lodash": "^4.17.21",
"make-plural": "^7.2.0",
"mousetrap": "^1.6.5",
"react": "^18.2.0",
"react-async-hook": "^4.0.0",
"react-contexify": "^6.0.0",
"react-dom": "^18.2.0",
"react-icons": "^4.7.1",
"react-infinite-scroller": "^1.2.6",
"react-redux": "^8.0.5",
"react-router-dom": "^6.8.0",
"react-swipeable": "^7.0.0",
"swagger-ui-react": "^4.15.5",
"tinycon": "^0.6.8",
"websocket-heartbeat-js": "^1.1.1"
},
"devDependencies": {
"@lingui/cli": "^3.17.0",
"@types/eslint": "^8.21.0",
"@types/lodash": "^4.14.191",
"@types/mousetrap": "^1.6.11",
"@types/react": "^18.0.27",
"@types/react-dom": "^18.0.10",
"@types/react-infinite-scroller": "^1.2.3",
"@types/swagger-ui-react": "^4.11.0",
"@types/tinycon": "^0.6.3",
"@typescript-eslint/eslint-plugin": "^5.50.0",
"@typescript-eslint/parser": "^5.50.0",
"@vitejs/plugin-react": "^3.1.0",
"eslint": "^8.33.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-typescript": "^17.0.0",
"eslint-config-prettier": "^8.6.0",
"eslint-config-react-app": "^7.0.1",
"eslint-plugin-hooks": "^0.4.3",
"eslint-plugin-prettier": "^4.2.1",
"prettier": "^2.8.3",
"rollup-plugin-visualizer": "^5.9.0",
"typescript": "^4.9.5",
"vite": "^4.1.1",
"vite-plugin-eslint": "^1.8.1",
"vite-tsconfig-paths": "^4.0.5",
"vitest": "^0.28.4",
"vitest-mock-extended": "^1.0.9"
}
"name": "commafeed-client",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite --host",
"dev:typescript": "tsc --watch",
"build": "npm run i18n:compile && tsc && vite build",
"preview": "vite preview",
"test": "vitest",
"test:ci": "vitest run",
"eslint": "eslint --ext=.js,.jsx,.ts,.tsx src",
"i18n": "npm run i18n:extract && npm run i18n:compile",
"i18n:extract": "lingui extract --clean",
"i18n:compile": "lingui compile --typescript",
"postinstall": "npm run i18n:compile"
},
"dependencies": {
"@emotion/react": "^11.10.5",
"@fontsource/open-sans": "^4.5.14",
"@lingui/core": "^3.17.0",
"@lingui/macro": "^3.17.0",
"@lingui/react": "^3.17.0",
"@mantine/core": "^5.10.3",
"@mantine/form": "^5.10.3",
"@mantine/hooks": "^5.10.3",
"@mantine/modals": "^5.10.3",
"@mantine/notifications": "^5.10.3",
"@mantine/spotlight": "^5.10.3",
"@mantine/styles": "^5.10.3",
"@reduxjs/toolkit": "^1.9.2",
"axios": "^1.3.2",
"dayjs": "^1.11.7",
"interweave": "^13.0.0",
"lodash": "^4.17.21",
"make-plural": "^7.2.0",
"mousetrap": "^1.6.5",
"react": "^18.2.0",
"react-async-hook": "^4.0.0",
"react-contexify": "^6.0.0",
"react-dom": "^18.2.0",
"react-icons": "^4.7.1",
"react-infinite-scroller": "^1.2.6",
"react-redux": "^8.0.5",
"react-router-dom": "^6.8.0",
"react-swipeable": "^7.0.0",
"swagger-ui-react": "^4.15.5",
"tinycon": "^0.6.8",
"use-local-storage": "^3.0.0",
"websocket-heartbeat-js": "^1.1.1"
},
"devDependencies": {
"@lingui/cli": "^3.17.0",
"@types/eslint": "^8.21.0",
"@types/lodash": "^4.14.191",
"@types/mousetrap": "^1.6.11",
"@types/react": "^18.0.27",
"@types/react-dom": "^18.0.10",
"@types/react-infinite-scroller": "^1.2.3",
"@types/swagger-ui-react": "^4.11.0",
"@types/tinycon": "^0.6.3",
"@typescript-eslint/eslint-plugin": "^5.50.0",
"@typescript-eslint/parser": "^5.50.0",
"@vitejs/plugin-react": "^3.1.0",
"eslint": "^8.33.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-typescript": "^17.0.0",
"eslint-config-prettier": "^8.6.0",
"eslint-config-react-app": "^7.0.1",
"eslint-plugin-hooks": "^0.4.3",
"eslint-plugin-prettier": "^4.2.1",
"prettier": "^2.8.3",
"rollup-plugin-visualizer": "^5.9.0",
"typescript": "^4.9.5",
"vite": "^4.1.1",
"vite-plugin-eslint": "^1.8.1",
"vite-tsconfig-paths": "^4.0.5",
"vitest": "^0.28.4",
"vitest-mock-extended": "^1.0.9"
}
}

View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId>
<version>3.0.0</version>
<version>3.1.0</version>
</parent>
<artifactId>commafeed-client</artifactId>
<name>CommaFeed Client</name>

View File

@@ -1,7 +1,8 @@
import { i18n } from "@lingui/core"
import { I18nProvider } from "@lingui/react"
import { ColorScheme, ColorSchemeProvider, MantineProvider } from "@mantine/core"
import { useColorScheme, useLocalStorage } from "@mantine/hooks"
import { useColorScheme } from "@mantine/hooks"
import useLocalStorage from "use-local-storage"
import { ModalsProvider } from "@mantine/modals"
import { NotificationsProvider } from "@mantine/notifications"
import { Constants } from "app/constants"
@@ -32,11 +33,7 @@ import Tinycon from "tinycon"
function Providers(props: { children: React.ReactNode }) {
const preferredColorScheme = useColorScheme()
const [colorScheme, setColorScheme] = useLocalStorage<ColorScheme>({
key: "color-scheme",
defaultValue: preferredColorScheme,
getInitialValueInEffect: true,
})
const [colorScheme, setColorScheme] = useLocalStorage<ColorScheme>("color-scheme", preferredColorScheme)
const toggleColorScheme = (value?: ColorScheme) => setColorScheme(value || (colorScheme === "dark" ? "light" : "dark"))
return (

View File

@@ -3,7 +3,7 @@ import { showNotification } from "@mantine/notifications"
import { createAsyncThunk, createSlice, isAnyOf } from "@reduxjs/toolkit"
import { client } from "app/client"
import { RootState } from "app/store"
import { ReadingMode, ReadingOrder, Settings, SharingSettings, UserModel, ViewMode } from "app/types"
import { ReadingMode, ReadingOrder, Settings, SharingSettings, UserModel } from "app/types"
// eslint-disable-next-line import/no-cycle
import { reloadEntries } from "./entries"
@@ -36,46 +36,67 @@ export const changeReadingOrder = createAsyncThunk<void, ReadingOrder, { state:
thunkApi.dispatch(reloadEntries())
}
)
export const changeViewMode = createAsyncThunk<void, ViewMode, { state: RootState }>("settings/viewMode", (viewMode, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({ ...settings, viewMode })
thunkApi.dispatch(reloadEntries())
})
export const changeLanguage = createAsyncThunk<void, string, { state: RootState }>("settings/language", (language, thunkApi) => {
export const changeLanguage = createAsyncThunk<
void,
string,
{
state: RootState
}
>("settings/language", (language, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({ ...settings, language })
})
export const changeScrollSpeed = createAsyncThunk<void, boolean, { state: RootState }>("settings/scrollSpeed", (speed, thunkApi) => {
export const changeScrollSpeed = createAsyncThunk<
void,
boolean,
{
state: RootState
}
>("settings/scrollSpeed", (speed, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({ ...settings, scrollSpeed: speed ? 400 : 0 })
})
export const changeShowRead = createAsyncThunk<void, boolean, { state: RootState }>("settings/showRead", (showRead, thunkApi) => {
export const changeShowRead = createAsyncThunk<
void,
boolean,
{
state: RootState
}
>("settings/showRead", (showRead, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({ ...settings, showRead })
})
export const changeScrollMarks = createAsyncThunk<void, boolean, { state: RootState }>("settings/scrollMarks", (scrollMarks, thunkApi) => {
export const changeScrollMarks = createAsyncThunk<
void,
boolean,
{
state: RootState
}
>("settings/scrollMarks", (scrollMarks, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({ ...settings, scrollMarks })
})
export const changeSharingSetting = createAsyncThunk<void, { site: keyof SharingSettings; value: boolean }, { state: RootState }>(
"settings/sharingSetting",
(sharingSetting, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({
...settings,
sharingSettings: {
...settings.sharingSettings,
[sharingSetting.site]: sharingSetting.value,
},
})
export const changeSharingSetting = createAsyncThunk<
void,
{ site: keyof SharingSettings; value: boolean },
{
state: RootState
}
)
>("settings/sharingSetting", (sharingSetting, thunkApi) => {
const { settings } = thunkApi.getState().user
if (!settings) return
client.user.saveSettings({
...settings,
sharingSettings: {
...settings.sharingSettings,
[sharingSetting.site]: sharingSetting.value,
},
})
})
export const userSlice = createSlice({
name: "user",
@@ -99,10 +120,6 @@ export const userSlice = createSlice({
if (!state.settings) return
state.settings.readingOrder = action.meta.arg
})
builder.addCase(changeViewMode.pending, (state, action) => {
if (!state.settings) return
state.settings.viewMode = action.meta.arg
})
builder.addCase(changeLanguage.pending, (state, action) => {
if (!state.settings) return
state.settings.language = action.meta.arg

View File

@@ -228,7 +228,6 @@ export interface Settings {
language: string
readingMode: ReadingMode
readingOrder: ReadingOrder
viewMode: ViewMode
showRead: boolean
scrollMarks: boolean
theme?: string
@@ -306,4 +305,4 @@ export type ReadingMode = "all" | "unread"
export type ReadingOrder = "asc" | "desc"
export type ViewMode = "title" | "cozy" | "expanded"
export type ViewMode = "title" | "cozy" | "detailed" | "expanded"

View File

@@ -21,6 +21,7 @@ import throttle from "lodash/throttle"
import { useEffect } from "react"
import InfiniteScroll from "react-infinite-scroller"
import { FeedEntry } from "./FeedEntry"
import { useViewMode } from "../../hooks/useViewMode"
export function FeedEntries() {
const source = useAppSelector(state => state.entries.source)
@@ -28,7 +29,7 @@ export function FeedEntries() {
const entriesTimestamp = useAppSelector(state => state.entries.timestamp)
const selectedEntryId = useAppSelector(state => state.entries.selectedEntryId)
const hasMore = useAppSelector(state => state.entries.hasMore)
const viewMode = useAppSelector(state => state.user.settings?.viewMode)
const { viewMode } = useViewMode()
const scrollMarks = useAppSelector(state => state.user.settings?.scrollMarks)
const scrollingToEntry = useAppSelector(state => state.entries.scrollingToEntry)
const dispatch = useAppDispatch()

View File

@@ -1,15 +1,17 @@
import { Anchor, Box, createStyles, Divider, Paper } from "@mantine/core"
import { Constants } from "app/constants"
import { markEntry } from "app/slices/entries"
import { useAppDispatch, useAppSelector } from "app/store"
import { Entry } from "app/types"
import { useAppDispatch } from "app/store"
import { Entry, ViewMode } from "app/types"
import React from "react"
import { useSwipeable } from "react-swipeable"
import { MantineNumberSize } from "@mantine/styles"
import { FeedEntryBody } from "./FeedEntryBody"
import { FeedEntryCompactHeader } from "./FeedEntryCompactHeader"
import { FeedEntryContextMenu, useFeedEntryContextMenu } from "./FeedEntryContextMenu"
import { FeedEntryFooter } from "./FeedEntryFooter"
import { FeedEntryHeader } from "./FeedEntryHeader"
import { useViewMode } from "../../hooks/useViewMode"
interface FeedEntryProps {
entry: Entry
@@ -18,16 +20,23 @@ interface FeedEntryProps {
onHeaderClick: (e: React.MouseEvent) => void
}
const useStyles = createStyles((theme, props: FeedEntryProps & { compact: boolean }) => {
const useStyles = createStyles((theme, props: FeedEntryProps & { viewMode?: ViewMode }) => {
let backgroundColor
if (theme.colorScheme === "dark") backgroundColor = props.entry.read ? "inherit" : theme.colors.dark[5]
else backgroundColor = props.entry.read && !props.expanded ? theme.colors.gray[0] : "inherit"
let marginY = theme.spacing.xs
if (props.compact) marginY = 6
if (props.viewMode === "title") marginY = 2
else if (props.viewMode === "cozy") marginY = 6
let mobileMarginY = 6
if (props.compact) mobileMarginY = 4
if (props.viewMode === "title") mobileMarginY = 2
else if (props.viewMode === "cozy") mobileMarginY = 4
let backgroundHoverColor = backgroundColor
if (!props.expanded) {
backgroundHoverColor = theme.colorScheme === "dark" ? theme.colors.dark[6] : theme.colors.gray[1]
}
const styles = {
paper: {
@@ -38,6 +47,11 @@ const useStyles = createStyles((theme, props: FeedEntryProps & { compact: boolea
marginTop: mobileMarginY,
marginBottom: mobileMarginY,
},
"@media (hover: hover)": {
"&:hover": {
backgroundColor: backgroundHoverColor,
},
},
},
body: {
maxWidth: Constants.layout.entryMaxWidth,
@@ -52,11 +66,8 @@ const useStyles = createStyles((theme, props: FeedEntryProps & { compact: boolea
})
export function FeedEntry(props: FeedEntryProps) {
const viewMode = useAppSelector(state => state.user.settings?.viewMode)
const compact = viewMode === "title"
const compactHeader = compact && !props.expanded
const { classes } = useStyles({ ...props, compact })
const { viewMode } = useViewMode()
const { classes } = useStyles({ ...props, viewMode })
const dispatch = useAppDispatch()
@@ -66,9 +77,18 @@ export function FeedEntry(props: FeedEntryProps) {
const { onContextMenu } = useFeedEntryContextMenu(props.entry)
const spacing = compact ? 8 : "xs"
const borderRadius = compact ? "xs" : "sm"
let paddingX: MantineNumberSize = "xs"
if (viewMode === "title" || viewMode === "cozy") paddingX = 6
let paddingY: MantineNumberSize = "xs"
if (viewMode === "title") paddingY = 4
else if (viewMode === "cozy") paddingY = 8
let borderRadius: MantineNumberSize = "sm"
if (viewMode === "title") borderRadius = 0
else if (viewMode === "cozy") borderRadius = "xs"
const compactHeader = !props.expanded && (viewMode === "title" || viewMode === "cozy")
return (
<Paper withBorder radius={borderRadius} className={classes.paper}>
<Anchor
@@ -80,17 +100,17 @@ export function FeedEntry(props: FeedEntryProps) {
onAuxClick={props.onHeaderClick}
onContextMenu={onContextMenu}
>
<Box p={spacing} {...swipeHandlers}>
<Box px={paddingX} py={paddingY} {...swipeHandlers}>
{compactHeader && <FeedEntryCompactHeader entry={props.entry} />}
{!compactHeader && <FeedEntryHeader entry={props.entry} expanded={props.expanded} />}
</Box>
</Anchor>
{props.expanded && (
<Box px={spacing} pb={spacing}>
<Box px={paddingX} pb={paddingY}>
<Box className={classes.body} sx={{ direction: props.entry.rtl ? "rtl" : "ltr" }}>
<FeedEntryBody entry={props.entry} />
</Box>
<Divider variant="dashed" my={spacing} />
<Divider variant="dashed" my={paddingY} />
<FeedEntryFooter entry={props.entry} />
</Box>
)}

View File

@@ -3,7 +3,6 @@ import { Box, Divider, Group, Menu, SegmentedControl, SegmentedControlItem, useM
import { showNotification } from "@mantine/notifications"
import { client } from "app/client"
import { redirectToAbout, redirectToAdminUsers, redirectToMetrics, redirectToSettings } from "app/slices/redirect"
import { changeViewMode } from "app/slices/user"
import { useAppDispatch, useAppSelector } from "app/store"
import { ViewMode } from "app/types"
import { useState } from "react"
@@ -12,6 +11,7 @@ import {
TbHelp,
TbLayoutList,
TbList,
TbListDetails,
TbMoon,
TbNotes,
TbPower,
@@ -20,6 +20,7 @@ import {
TbUsers,
TbWorldDownload,
} from "react-icons/tb"
import { useViewMode } from "../../hooks/useViewMode"
interface ProfileMenuProps {
control: React.ReactElement
@@ -54,6 +55,17 @@ const viewModeData: ViewModeControlItem[] = [
</Group>
),
},
{
value: "detailed",
label: (
<Group>
<TbListDetails size={iconSize} />
<Box ml={6}>
<Trans>Detailed</Trans>
</Box>
</Group>
),
},
{
value: "expanded",
label: (
@@ -69,7 +81,7 @@ const viewModeData: ViewModeControlItem[] = [
export function ProfileMenu(props: ProfileMenuProps) {
const [opened, setOpened] = useState(false)
const viewMode = useAppSelector(state => state.user.settings?.viewMode)
const { viewMode, setViewMode } = useViewMode()
const profile = useAppSelector(state => state.user.profile)
const admin = useAppSelector(state => state.user.profile?.admin)
const dispatch = useAppDispatch()
@@ -127,7 +139,7 @@ export function ProfileMenu(props: ProfileMenuProps) {
orientation="vertical"
data={viewModeData}
value={viewMode}
onChange={e => dispatch(changeViewMode(e as ViewMode))}
onChange={e => setViewMode(e as ViewMode)}
mb="xs"
/>

View File

@@ -0,0 +1,7 @@
import useLocalStorage from "use-local-storage"
import { ViewMode } from "../app/types"
export function useViewMode() {
const [viewMode, setViewMode] = useLocalStorage<ViewMode>("view-mode", "detailed")
return { viewMode, setViewMode }
}

View File

@@ -219,6 +219,10 @@ msgstr "حذف المستخدم"
msgid "Desc"
msgstr "تنازلي"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Suprimeix l'usuari"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Smazat uživatele"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Dileu defnyddiwr"
msgid "Desc"
msgstr "Rhag"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Slet bruger"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Benutzer löschen"
msgid "Desc"
msgstr "Beschr"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Delete user"
msgid "Desc"
msgstr "Desc"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr "Detailed"
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Borrar usuario"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "حذف کاربر"
msgid "Desc"
msgstr "توصیف"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Poista käyttäjä"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Effacer l'utilisateur"
msgid "Desc"
msgstr "Descendant"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr "Vue détaillée"
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Eliminar usuario"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Felhasználó törlése"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Hapus pengguna"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Elimina utente"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "ユーザーの削除"
msgid "Desc"
msgstr "説明"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "사용자 삭제"
msgid "Desc"
msgstr "설명"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Padam pengguna"
msgid "Desc"
msgstr "Dec"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Slett bruker"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Gebruiker verwijderen"
msgid "Desc"
msgstr "Beschrijving"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Slett bruker"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Usuń użytkownika"
msgid "Desc"
msgstr "Opis"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Excluir usuário"
msgid "Desc"
msgstr "Descrição"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Удалить пользователя"
msgid "Desc"
msgstr "По убыванию"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Vymažte používateľa"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Ta bort användare"
msgid "Desc"
msgstr ""
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "Kullanıcıyı sil"
msgid "Desc"
msgstr "Açılış"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -219,6 +219,10 @@ msgstr "删除用户"
msgid "Desc"
msgstr "描述"
#: src/components/header/ProfileMenu.tsx
msgid "Detailed"
msgstr ""
#: src/components/header/ProfileMenu.tsx
#: src/pages/app/SettingsPage.tsx
msgid "Display"

View File

@@ -1,7 +1,6 @@
import { Accordion, Box, Tabs } from "@mantine/core"
import { Accordion, Tabs } from "@mantine/core"
import { client } from "app/client"
import { Loader } from "components/Loader"
import { Gauge } from "components/metrics/Gauge"
import { Meter } from "components/metrics/Meter"
import { MetricAccordionItem } from "components/metrics/MetricAccordionItem"
import { Timer } from "components/metrics/Timer"
@@ -9,26 +8,18 @@ import { useAsync } from "react-async-hook"
import { TbChartAreaLine, TbClock } from "react-icons/tb"
const shownMeters: { [key: string]: string } = {
"com.commafeed.backend.feed.FeedQueues.refill": "Refresh queue refill rate",
"com.commafeed.backend.feed.FeedRefreshTaskGiver.feedRefreshed": "Feed refreshed",
"com.commafeed.backend.feed.FeedRefreshUpdater.feedUpdated": "Feed updated",
"com.commafeed.backend.feed.FeedRefreshUpdater.entryCacheHit": "Entry cache hit",
"com.commafeed.backend.feed.FeedRefreshUpdater.entryCacheMiss": "Entry cache miss",
}
const shownGauges: { [key: string]: string } = {
"com.commafeed.backend.feed.FeedRefreshExecutor.feed-refresh-updater.active": "Feed Updater active",
"com.commafeed.backend.feed.FeedRefreshExecutor.feed-refresh-updater.pending": "Feed Updater queued",
"com.commafeed.backend.feed.FeedRefreshExecutor.feed-refresh-worker.active": "Feed Worker active",
"com.commafeed.backend.feed.FeedRefreshExecutor.feed-refresh-worker.pending": "Feed Worker queued",
"com.commafeed.backend.feed.FeedQueues.queue": "Feed Refresh queue size",
"com.commafeed.backend.feed.FeedRefreshEngine.refill": "Feed queue refill rate",
"com.commafeed.backend.feed.FeedRefreshWorker.feedFetched": "Feed fetching rate",
"com.commafeed.backend.feed.FeedRefreshUpdater.feedUpdated": "Feed update rate",
"com.commafeed.backend.feed.FeedRefreshUpdater.entryCacheHit": "Entry cache hit rate",
"com.commafeed.backend.feed.FeedRefreshUpdater.entryCacheMiss": "Entry cache miss rate",
}
export function MetricsPage() {
const query = useAsync(() => client.admin.getMetrics(), [])
if (!query.result) return <Loader />
const { meters, gauges, timers } = query.result.data
const { meters, timers } = query.result.data
return (
<Tabs defaultValue="stats">
<Tabs.List>
@@ -48,15 +39,6 @@ export function MetricsPage() {
</MetricAccordionItem>
))}
</Accordion>
<Box pt="xs">
{Object.keys(shownGauges).map(g => (
<Box key={g}>
<span>{shownGauges[g]}&nbsp;</span>
<Gauge gauge={gauges[g]} />
</Box>
))}
</Box>
</Tabs.Panel>
<Tabs.Panel value="timers" pt="xs">

View File

@@ -69,7 +69,7 @@ app:
# user-agent string that will be used by the http client, leave empty for the default one
userAgent:
# Database connection
# -------------------
# for MySQL
@@ -92,7 +92,7 @@ database:
properties:
charSet: UTF-8
validationQuery: "/* CommaFeed Health Check */ SELECT 1"
server:
applicationConnectors:
- type: http
@@ -100,7 +100,7 @@ server:
adminConnectors:
- type: http
port: 8084
logging:
level: INFO
loggers:
@@ -108,6 +108,7 @@ logging:
liquibase: INFO
org.hibernate.SQL: INFO # or ALL for sql debugging
org.hibernate.engine.internal.StatisticalLoggingSessionEventListener: WARN
org.hibernate.orm.deprecation: "OFF"
appenders:
- type: console
- type: file
@@ -117,14 +118,14 @@ logging:
archivedLogFilenamePattern: log/commafeed-%d.log
archivedFileCount: 5
timeZone: UTC
# Redis pool configuration
# (only used if app.cache is 'redis')
# -----------------------------------
redis:
host: localhost
port: 6379
password:
password:
timeout: 2000
database: 0
maxTotal: 500

View File

@@ -104,9 +104,11 @@ server:
adminConnectors:
- type: http
port: 8084
requestLog:
appenders: [ ]
logging:
level: WARN
level: ERROR
loggers:
com.commafeed: INFO
liquibase: INFO

View File

@@ -1,19 +1,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId>
<version>3.0.0</version>
<version>3.1.0</version>
</parent>
<artifactId>commafeed-server</artifactId>
<name>CommaFeed Server</name>
<properties>
<guice.version>5.1.0</guice.version>
<querydsl.version>4.2.1</querydsl.version>
<rome.version>1.18.0</rome.version>
<querydsl.version>4.4.0</querydsl.version>
<rome.version>2.1.0</rome.version>
</properties>
<dependencyManagement>
@@ -21,7 +22,7 @@
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-dependencies</artifactId>
<version>2.1.1</version>
<version>2.1.6</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@@ -112,8 +113,10 @@
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.commafeed.CommaFeedApplication</mainClass>
</transformer>
<transformer implementation="org.kordamp.shade.resources.PropertiesFileTransformer">
@@ -230,7 +233,7 @@
<dependency>
<groupId>com.commafeed</groupId>
<artifactId>commafeed-client</artifactId>
<version>3.0.0</version>
<version>3.1.0</version>
</dependency>
<dependency>
@@ -290,7 +293,7 @@
<dependency>
<groupId>be.tomcools</groupId>
<artifactId>dropwizard-websocket-jee7-bundle</artifactId>
<version>2.0.0</version>
<version>2.1.6</version>
</dependency>
<dependency>
<groupId>io.whitfin</groupId>
@@ -298,6 +301,12 @@
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava3</groupId>
<artifactId>rxjava</artifactId>
<version>3.1.6</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
@@ -410,50 +419,38 @@
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.3</version>
<version>1.15.4</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>70.1</version>
<version>73.1</version>
</dependency>
<dependency>
<groupId>net.sourceforge.cssparser</groupId>
<artifactId>cssparser</artifactId>
<version>0.9.29</version>
<version>0.9.30</version>
</dependency>
<dependency>
<groupId>edu.uci.ics</groupId>
<artifactId>crawler4j</artifactId>
<version>3.5</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
<groupId>org.netpreserve</groupId>
<artifactId>urlcanon</artifactId>
<version>0.4.0</version>
</dependency>
<dependency>
<groupId>com.google.gwt</groupId>
<artifactId>gwt-servlet</artifactId>
<version>2.9.0</version>
<version>2.10.0</version>
</dependency>
<dependency>
<groupId>io.github.hakky54</groupId>
<artifactId>sslcontext-kickstart</artifactId>
<version>7.2.0</version>
<version>7.4.11</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-youtube</artifactId>
<version>v3-rev139-1.20.0</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
<version>v3-rev222-1.25.0</version>
</dependency>
<dependency>
@@ -463,12 +460,12 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.28</version>
<version>8.0.33</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.4.1</version>
<version>42.6.0</version>
</dependency>
<dependency>
<groupId>net.sourceforge.jtds</groupId>
@@ -494,7 +491,7 @@
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-junit-jupiter</artifactId>
<version>5.13.2</version>
<version>5.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -510,7 +507,7 @@
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
<version>1.24.1</version>
<version>1.32.0</version>
<scope>test</scope>
</dependency>

View File

@@ -18,9 +18,7 @@ import javax.websocket.server.ServerEndpointConfig;
import org.hibernate.cfg.AvailableSettings;
import com.codahale.metrics.json.MetricsModule;
import com.commafeed.backend.feed.FeedRefreshTaskGiver;
import com.commafeed.backend.feed.FeedRefreshUpdater;
import com.commafeed.backend.feed.FeedRefreshWorker;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
@@ -32,7 +30,7 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.StartupService;
import com.commafeed.backend.service.DatabaseStartupService;
import com.commafeed.backend.service.UserService;
import com.commafeed.backend.task.ScheduledTask;
import com.commafeed.frontend.auth.SecurityCheckFactoryProvider;
@@ -95,7 +93,8 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
@Override
protected ObjectMapper configureObjectMapper(ObjectMapper objectMapper) {
// disable case sensitivity because EnvironmentSubstitutor maps MYPROPERTY to myproperty and not to myProperty
return objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES);
return objectMapper
.setConfig(objectMapper.getDeserializationConfig().with(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES));
}
});
@@ -194,12 +193,10 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
}
// database init/changelogs
environment.lifecycle().manage(injector.getInstance(StartupService.class));
environment.lifecycle().manage(injector.getInstance(DatabaseStartupService.class));
// background feed fetching
environment.lifecycle().manage(injector.getInstance(FeedRefreshTaskGiver.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshWorker.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshUpdater.class));
// start feed fetching engine
environment.lifecycle().manage(injector.getInstance(FeedRefreshEngine.class));
// prevent caching index.html, so that the webapp is always up to date
environment.servlets()

View File

@@ -14,7 +14,6 @@ import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
@@ -59,7 +58,7 @@ public class HttpGetter {
}
}
public HttpResult getBinary(String url, int timeout) throws ClientProtocolException, IOException, NotModifiedException {
public HttpResult getBinary(String url, int timeout) throws IOException, NotModifiedException {
return getBinary(url, null, null, timeout);
}
@@ -71,14 +70,10 @@ public class HttpGetter {
* header we got last time we queried that url, or null
* @param eTag
* header we got last time we queried that url, or null
* @return
* @throws ClientProtocolException
* @throws IOException
* @throws NotModifiedException
* if the url hasn't changed since we asked for it last time
*/
public HttpResult getBinary(String url, String lastModified, String eTag, int timeout)
throws ClientProtocolException, IOException, NotModifiedException {
public HttpResult getBinary(String url, String lastModified, String eTag, int timeout) throws IOException, NotModifiedException {
HttpResult result = null;
long start = System.currentTimeMillis();
@@ -175,13 +170,6 @@ public class HttpGetter {
return builder.build();
}
public static void main(String[] args) throws Exception {
CommaFeedConfiguration config = new CommaFeedConfiguration();
HttpGetter getter = new HttpGetter(config);
HttpResult result = getter.getBinary("https://sourceforge.net/projects/mpv-player-windows/rss", 30000);
System.out.println(new String(result.content));
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
@@ -189,12 +177,12 @@ public class HttpGetter {
/**
* if the value of this header changed, this is its new value
*/
private String newLastModifiedHeader;
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private String newEtagHeader;
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);

View File

@@ -36,7 +36,7 @@ public class FeedDAO extends GenericDAO<Feed> {
QFeedSubscription subs = QFeedSubscription.feedSubscription;
QUser user = QUser.user;
query.join(feed.subscriptions, subs).join(subs.user, user).where(user.lastLogin.gt(lastLoginThreshold));
query.join(subs).on(subs.feed.id.eq(feed.id)).join(subs.user, user).where(user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(feed.disabledUntil.asc()).limit(count).fetch();

View File

@@ -2,6 +2,7 @@ package com.commafeed.backend.feed;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Set;
import javax.inject.Inject;
@@ -13,13 +14,19 @@ import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedParser.FeedParserResult;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
@@ -29,18 +36,18 @@ public class FeedFetcher {
private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders;
public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate,
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate,
String lastContentHash) throws FeedException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
int timeout = 20000;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag, timeout);
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, StringUtils.newStringUtf8(result.getContent()));
@@ -49,7 +56,7 @@ public class FeedFetcher {
result = getter.getBinary(extractedUrl, lastModified, eTag, timeout);
content = result.getContent();
fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw e;
}
@@ -73,21 +80,20 @@ public class FeedFetcher {
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
if (lastPublishedDate != null && parserResult.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == parserResult.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
Feed feed = fetchedFeed.getFeed();
Feed feed = parserResult.getFeed();
feed.setLastModifiedHeader(result.getLastModifiedSince());
feed.setEtagHeader(FeedUtils.truncate(result.getETag(), 255));
feed.setLastContentHash(hash);
fetchedFeed.setFetchDuration(result.getDuration());
fetchedFeed.setUrlAfterRedirect(result.getUrlAfterRedirect());
return fetchedFeed;
return new FeedFetcherResult(parserResult.getFeed(), parserResult.getEntries(), parserResult.getTitle(),
result.getUrlAfterRedirect(), result.getDuration());
}
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
@@ -100,4 +106,14 @@ public class FeedFetcher {
return null;
}
@Value
public static class FeedFetcherResult {
Feed feed;
List<FeedEntry> entries;
String title;
String urlAfterRedirect;
long fetchDuration;
}
}

View File

@@ -3,6 +3,7 @@ package com.commafeed.backend.feed;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@@ -37,8 +38,12 @@ import com.rometools.rome.io.SyndFeedInput;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
/**
* Parses raw xml as a Feed object
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
@@ -50,10 +55,7 @@ public class FeedParser {
private static final Date START = new Date(86400000);
private static final Date END = new Date(1000L * Integer.MAX_VALUE - 86400000);
public FetchedFeed parse(String feedUrl, byte[] xml) throws FeedException {
FetchedFeed fetchedFeed = new FetchedFeed();
Feed feed = fetchedFeed.getFeed();
List<FeedEntry> entries = fetchedFeed.getEntries();
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedException {
try {
Charset encoding = FeedUtils.guessEncoding(xml);
@@ -63,17 +65,19 @@ public class FeedParser {
}
xmlString = FeedUtils.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed rss = new SyndFeedInput().build(source);
handleForeignMarkup(rss);
fetchedFeed.setTitle(rss.getTitle());
String title = rss.getTitle();
Feed feed = new Feed();
feed.setPushHub(findHub(rss));
feed.setPushTopic(findSelf(rss));
feed.setUrl(feedUrl);
feed.setLink(rss.getLink());
List<SyndEntry> items = rss.getEntries();
for (SyndEntry item : items) {
List<FeedEntry> entries = new ArrayList<>();
for (SyndEntry item : rss.getEntries()) {
FeedEntry entry = new FeedEntry();
String guid = item.getUri();
@@ -121,6 +125,7 @@ public class FeedParser {
entries.add(entry);
}
Date lastEntryDate = null;
Date publishedDate = validateDate(rss.getPublishedDate(), false);
if (!entries.isEmpty()) {
@@ -133,10 +138,10 @@ public class FeedParser {
feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
feed.setLastEntryDate(lastEntryDate);
return new FeedParserResult(feed, entries, title);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
return fetchedFeed;
}
/**
@@ -273,4 +278,11 @@ public class FeedParser {
}
}
@Value
public static class FeedParserResult {
Feed feed;
List<FeedEntry> entries;
String title;
}
}

View File

@@ -1,124 +0,0 @@
package com.commafeed.backend.feed;
import java.util.Date;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
@Singleton
public class FeedQueues {
private final SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final CommaFeedConfiguration config;
private final BlockingDeque<FeedRefreshContext> queue = new LinkedBlockingDeque<>();
private final Meter refill;
@Inject
public FeedQueues(SessionFactory sessionFactory, FeedDAO feedDAO, CommaFeedConfiguration config, MetricRegistry metrics) {
this.sessionFactory = sessionFactory;
this.config = config;
this.feedDAO = feedDAO;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
metrics.register(MetricRegistry.name(getClass(), "queue"), (Gauge<Integer>) queue::size);
}
/**
* take a feed from the refresh queue
*/
public synchronized FeedRefreshContext take() {
FeedRefreshContext context = queue.poll();
if (context != null) {
return context;
}
refill();
try {
// try to get something from the queue
// if the queue is empty, wait a bit
// polling the queue instead of sleeping gives us the opportunity to process a feed immediately if it was added manually with
// add()
return queue.poll(15, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw new RuntimeException("interrupted while waiting for a feed in the queue", e);
}
}
/**
* add a feed to the refresh queue
*/
public void add(Feed feed, boolean urgent) {
if (isFeedAlreadyQueued(feed)) {
return;
}
FeedRefreshContext context = new FeedRefreshContext(feed, urgent);
if (urgent) {
queue.addFirst(context);
} else {
queue.addLast(context);
}
}
/**
* refills the refresh queue
*/
private void refill() {
refill.mark();
// add feeds that are up to refresh from the database
int batchSize = Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> {
List<Feed> list = feedDAO.findNextUpdatable(batchSize, getLastLoginThreshold());
// set the disabledDate as we use it in feedDAO.findNextUpdatable() to decide what to refresh next
Date nextRefreshDate = DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes());
list.forEach(f -> f.setDisabledUntil(nextRefreshDate));
feedDAO.saveOrUpdate(list);
return list;
});
feeds.forEach(f -> add(f, false));
}
public void giveBack(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
UnitOfWork.run(sessionFactory, () -> feedDAO.saveOrUpdate(feed));
// we just finished updating the feed, remove it from the queue
queue.removeIf(c -> isFeedAlreadyQueued(c.getFeed()));
}
private Date getLastLoginThreshold() {
if (config.getApplicationSettings().getHeavyLoad()) {
return DateUtils.addDays(new Date(), -30);
} else {
return null;
}
}
private boolean isFeedAlreadyQueued(Feed feed) {
return queue.stream().anyMatch(c -> c.getFeed().getId().equals(feed.getId()));
}
}

View File

@@ -1,22 +0,0 @@
package com.commafeed.backend.feed;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FeedRefreshContext {
private Feed feed;
private List<FeedEntry> entries;
private boolean urgent;
public FeedRefreshContext(Feed feed, boolean isUrgent) {
this.feed = feed;
this.urgent = isUrgent;
}
}

View File

@@ -0,0 +1,108 @@
package com.commafeed.backend.feed;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
import io.dropwizard.lifecycle.Managed;
import io.reactivex.rxjava3.core.Flowable;
import io.reactivex.rxjava3.disposables.Disposable;
import io.reactivex.rxjava3.processors.PublishProcessor;
import io.reactivex.rxjava3.schedulers.Schedulers;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine implements Managed {
private final SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final PublishProcessor<Feed> priorityQueue;
private Disposable flow;
@Inject
public FeedRefreshEngine(SessionFactory sessionFactory, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.sessionFactory = sessionFactory;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.priorityQueue = PublishProcessor.create();
}
@Override
public void start() {
Flowable<Feed> database = Flowable.fromCallable(() -> findNextUpdatableFeeds(getBatchSize(), getLastLoginThreshold()))
.onErrorResumeNext(e -> {
log.error("error while fetching next updatable feeds", e);
return Flowable.empty();
})
// repeat query 15s after the flowable has been emptied
// https://github.com/ReactiveX/RxJava/issues/448#issuecomment-233244964
.repeatWhen(o -> o.concatMap(v -> Flowable.timer(15, TimeUnit.SECONDS)))
.flatMap(Flowable::fromIterable);
Flowable<Feed> source = Flowable.merge(priorityQueue, database);
this.flow = source.subscribeOn(Schedulers.io())
// feed fetching
.parallel(config.getApplicationSettings().getBackgroundThreads())
.runOn(Schedulers.io())
.flatMap(f -> Flowable.fromCallable(() -> worker.update(f)).onErrorResumeNext(e -> {
log.error("error while fetching feed", e);
return Flowable.empty();
}))
.sequential()
// database updating
.parallel(config.getApplicationSettings().getDatabaseUpdateThreads())
.runOn(Schedulers.io())
.flatMap(fae -> Flowable.fromCallable(() -> updater.update(fae.getFeed(), fae.getEntries())).onErrorResumeNext(e -> {
log.error("error while updating database", e);
return Flowable.empty();
}))
.sequential()
// end flow
.subscribe();
}
public void refreshImmediately(Feed feed) {
priorityQueue.onNext(feed);
}
private List<Feed> findNextUpdatableFeeds(int max, Date lastLoginThreshold) {
refill.mark();
return UnitOfWork.call(sessionFactory, () -> feedDAO.findNextUpdatable(max, lastLoginThreshold));
}
private int getBatchSize() {
return Math.min(Flowable.bufferSize(), 3 * config.getApplicationSettings().getBackgroundThreads());
}
private Date getLastLoginThreshold() {
return Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad()) ? DateUtils.addDays(new Date(), -30) : null;
}
@Override
public void stop() {
flow.dispose();
}
}

View File

@@ -1,99 +0,0 @@
package com.commafeed.backend.feed;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import lombok.extern.slf4j.Slf4j;
/**
* Wraps a {@link ThreadPoolExecutor} instance. Blocks when queue is full instead of rejecting the task. Allow priority queueing by using
* {@link Task} instead of {@link Runnable}
*
*/
@Slf4j
public class FeedRefreshExecutor {
private String poolName;
private ThreadPoolExecutor pool;
private LinkedBlockingDeque<Runnable> queue;
public FeedRefreshExecutor(final String poolName, int threads, int queueCapacity, MetricRegistry metrics) {
log.info("Creating pool {} with {} threads", poolName, threads);
this.poolName = poolName;
pool = new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS, queue = new LinkedBlockingDeque<Runnable>(queueCapacity) {
private static final long serialVersionUID = 1L;
@Override
public boolean offer(Runnable r) {
Task task = (Task) r;
if (task.isUrgent()) {
return offerFirst(r);
} else {
return offerLast(r);
}
}
}) {
@Override
protected void afterExecute(Runnable r, Throwable t) {
if (t != null) {
log.error("thread from pool {} threw a runtime exception", poolName, t);
}
}
};
pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
log.debug("{} thread queue full, waiting...", poolName);
try {
Task task = (Task) r;
if (task.isUrgent()) {
queue.putFirst(r);
} else {
queue.put(r);
}
} catch (InterruptedException e1) {
log.error(poolName + " interrupted while waiting for queue.", e1);
}
}
});
metrics.register(MetricRegistry.name(getClass(), poolName, "active"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return pool.getActiveCount();
}
});
metrics.register(MetricRegistry.name(getClass(), poolName, "pending"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return queue.size();
}
});
}
public void execute(Task task) {
pool.execute(task);
}
public void shutdown() {
pool.shutdownNow();
while (!pool.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("{} interrupted while waiting for threads to finish.", poolName);
}
}
}
public interface Task extends Runnable {
boolean isUrgent();
}
}

View File

@@ -22,9 +22,9 @@ public class FeedRefreshIntervalCalculator {
this.refreshIntervalMinutes = config.getApplicationSettings().getRefreshIntervalMinutes();
}
public Date onFetchSuccess(FetchedFeed fetchedFeed) {
public Date onFetchSuccess(Feed feed) {
Date defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(fetchedFeed.getFeed(), defaultRefreshInterval) : defaultRefreshInterval;
return heavyLoad ? computeRefreshIntervalForHeavyLoad(feed, defaultRefreshInterval) : defaultRefreshInterval;
}
public Date onFeedNotModified(Feed feed) {

View File

@@ -1,75 +0,0 @@
package com.commafeed.backend.feed;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
/**
* Infinite loop fetching feeds from @FeedQueues and queuing them to the {@link FeedRefreshWorker} pool.
*
*/
@Slf4j
@Singleton
public class FeedRefreshTaskGiver implements Managed {
private final FeedQueues queues;
private final FeedRefreshWorker worker;
private final ExecutorService executor;
private final Meter feedRefreshed;
@Inject
public FeedRefreshTaskGiver(FeedQueues queues, FeedDAO feedDAO, FeedRefreshWorker worker, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.queues = queues;
this.worker = worker;
executor = Executors.newFixedThreadPool(1);
feedRefreshed = metrics.meter(MetricRegistry.name(getClass(), "feedRefreshed"));
}
@Override
public void stop() {
log.info("shutting down feed refresh task giver");
executor.shutdownNow();
while (!executor.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("interrupted while waiting for threads to finish.");
}
}
}
@Override
public void start() {
log.info("starting feed refresh task giver");
executor.execute(new Runnable() {
@Override
public void run() {
while (!executor.isShutdown()) {
try {
FeedRefreshContext context = queues.take();
if (context != null) {
feedRefreshed.mark();
worker.updateFeed(context);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
}
});
}
}

View File

@@ -20,17 +20,16 @@ import org.hibernate.SessionFactory;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedUpdateService;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.backend.service.PubSubService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
@@ -40,20 +39,22 @@ import io.dropwizard.lifecycle.Managed;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater implements Managed {
private final SessionFactory sessionFactory;
private final FeedUpdateService feedUpdateService;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final PubSubService pubSubService;
private final FeedQueues queues;
private final CommaFeedConfiguration config;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final CacheService cache;
private final WebSocketSessions webSocketSessions;
private final FeedRefreshExecutor pool;
private final Striped<Lock> locks;
private final Meter entryCacheMiss;
@@ -62,22 +63,19 @@ public class FeedRefreshUpdater implements Managed {
private final Meter entryInserted;
@Inject
public FeedRefreshUpdater(SessionFactory sessionFactory, FeedUpdateService feedUpdateService, PubSubService pubSubService,
FeedQueues queues, CommaFeedConfiguration config, MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO,
public FeedRefreshUpdater(SessionFactory sessionFactory, FeedService feedService, FeedEntryService feedEntryService,
PubSubService pubSubService, CommaFeedConfiguration config, MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO,
CacheService cache, WebSocketSessions webSocketSessions) {
this.sessionFactory = sessionFactory;
this.feedUpdateService = feedUpdateService;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.pubSubService = pubSubService;
this.queues = queues;
this.config = config;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.cache = cache;
this.webSocketSessions = webSocketSessions;
ApplicationSettings settings = config.getApplicationSettings();
int threads = Math.max(settings.getDatabaseUpdateThreads(), 1);
pool = new FeedRefreshExecutor("feed-refresh-updater", threads, Math.min(50 * threads, 1000), metrics);
locks = Striped.lazyWeakLock(threads * 100000);
locks = Striped.lazyWeakLock(100000);
entryCacheMiss = metrics.meter(MetricRegistry.name(getClass(), "entryCacheMiss"));
entryCacheHit = metrics.meter(MetricRegistry.name(getClass(), "entryCacheHit"));
@@ -85,20 +83,6 @@ public class FeedRefreshUpdater implements Managed {
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
@Override
public void start() throws Exception {
}
@Override
public void stop() throws Exception {
log.info("shutting down feed refresh updater");
pool.shutdown();
}
public void updateFeed(FeedRefreshContext context) {
pool.execute(new EntryTask(context));
}
private AddEntryResult addEntry(final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
@@ -123,7 +107,7 @@ public class FeedRefreshUpdater implements Managed {
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = UnitOfWork.call(sessionFactory, () -> feedUpdateService.addEntry(feed, entry, subscriptions));
inserted = UnitOfWork.call(sessionFactory, () -> feedEntryService.addEntry(feed, entry, subscriptions));
if (inserted) {
entryInserted.mark();
}
@@ -166,76 +150,63 @@ public class FeedRefreshUpdater implements Managed {
}
}
private class EntryTask implements Task {
public boolean update(Feed feed, List<FeedEntry> entries) {
boolean processed = true;
boolean insertedAtLeastOneEntry = false;
private final FeedRefreshContext context;
if (!entries.isEmpty()) {
List<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>();
public EntryTask(FeedRefreshContext context) {
this.context = context;
}
@Override
public void run() {
boolean processed = true;
boolean insertedAtLeastOneEntry = false;
final Feed feed = context.getFeed();
List<FeedEntry> entries = context.getEntries();
if (entries.isEmpty()) {
feed.setMessage("Feed has no entries");
} else {
List<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>();
List<FeedSubscription> subscriptions = null;
for (FeedEntry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(feed, entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
if (subscriptions == null) {
subscriptions = UnitOfWork.call(sessionFactory, () -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
insertedAtLeastOneEntry |= addEntryResult.inserted;
entryCacheMiss.mark();
} else {
log.debug("cache hit for {}", entry.getUrl());
entryCacheHit.mark();
List<FeedSubscription> subscriptions = null;
for (FeedEntry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(feed, entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
if (subscriptions == null) {
subscriptions = UnitOfWork.call(sessionFactory, () -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
insertedAtLeastOneEntry |= addEntryResult.inserted;
currentEntries.add(cacheKey);
entryCacheMiss.mark();
} else {
log.debug("cache hit for {}", entry.getUrl());
entryCacheHit.mark();
}
cache.setLastEntries(feed, currentEntries);
if (subscriptions == null) {
feed.setMessage("No new entries found");
} else if (insertedAtLeastOneEntry) {
List<User> users = subscriptions.stream().map(FeedSubscription::getUser).collect(Collectors.toList());
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
currentEntries.add(cacheKey);
}
cache.setLastEntries(feed, currentEntries);
// notify over websocket
subscriptions.forEach(sub -> webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub)));
}
}
if (subscriptions == null) {
feed.setMessage("No new entries found");
} else if (insertedAtLeastOneEntry) {
List<User> users = subscriptions.stream().map(FeedSubscription::getUser).collect(Collectors.toList());
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
if (config.getApplicationSettings().getPubsubhubbub()) {
handlePubSub(feed);
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(new Date(0));
// notify over websocket
subscriptions.forEach(sub -> webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub)));
}
}
if (Boolean.TRUE.equals(config.getApplicationSettings().getPubsubhubbub())) {
handlePubSub(feed);
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(new Date(0));
}
if (insertedAtLeastOneEntry) {
feedUpdated.mark();
queues.giveBack(feed);
}
@Override
public boolean isUrgent() {
return context.isUrgent();
}
UnitOfWork.run(sessionFactory, () -> feedService.save(feed));
return processed;
}
@AllArgsConstructor

View File

@@ -1,5 +1,6 @@
package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -10,91 +11,72 @@ import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import io.dropwizard.lifecycle.Managed;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and handles its outcome
*
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker implements Managed {
public class FeedRefreshWorker {
private final FeedRefreshUpdater feedRefreshUpdater;
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final FeedQueues queues;
private final CommaFeedConfiguration config;
private final FeedRefreshExecutor pool;
private final Meter feedFetched;
@Inject
public FeedRefreshWorker(FeedRefreshUpdater feedRefreshUpdater, FeedRefreshIntervalCalculator refreshIntervalCalculator,
FeedFetcher fetcher, FeedQueues queues, CommaFeedConfiguration config, MetricRegistry metrics) {
this.feedRefreshUpdater = feedRefreshUpdater;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.queues = queues;
int threads = config.getApplicationSettings().getBackgroundThreads();
pool = new FeedRefreshExecutor("feed-refresh-worker", threads, Math.min(20 * threads, 1000), metrics);
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
@Override
public void start() throws Exception {
}
@Override
public void stop() throws Exception {
pool.shutdown();
}
public void updateFeed(FeedRefreshContext context) {
pool.execute(new FeedTask(context));
}
private void update(FeedRefreshContext context) {
Feed feed = context.getFeed();
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FetchedFeed fetchedFeed = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
FeedFetcherResult feedFetcherResult = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<FeedEntry> entries = fetchedFeed.getEntries();
List<FeedEntry> entries = feedFetcherResult.getEntries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).collect(Collectors.toList());
}
String urlAfterRedirect = fetchedFeed.getUrlAfterRedirect();
String urlAfterRedirect = feedFetcherResult.getUrlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(fetchedFeed.getFeed().getLink());
feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader());
feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
feed.setLastContentHash(fetchedFeed.getFeed().getLastContentHash());
feed.setLastPublishedDate(fetchedFeed.getFeed().getLastPublishedDate());
feed.setAverageEntryInterval(fetchedFeed.getFeed().getAverageEntryInterval());
feed.setLastEntryDate(fetchedFeed.getFeed().getLastEntryDate());
feed.setLink(feedFetcherResult.getFeed().getLink());
feed.setLastModifiedHeader(feedFetcherResult.getFeed().getLastModifiedHeader());
feed.setEtagHeader(feedFetcherResult.getFeed().getEtagHeader());
feed.setLastContentHash(feedFetcherResult.getFeed().getLastContentHash());
feed.setLastPublishedDate(feedFetcherResult.getFeed().getLastPublishedDate());
feed.setAverageEntryInterval(feedFetcherResult.getFeed().getAverageEntryInterval());
feed.setLastEntryDate(feedFetcherResult.getFeed().getLastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(fetchedFeed));
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(feedFetcherResult.getFeed()));
handlePubSub(feed, fetchedFeed.getFeed());
context.setEntries(entries);
feedRefreshUpdater.updateFeed(context);
handlePubSub(feed, feedFetcherResult.getFeed());
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
@@ -110,7 +92,7 @@ public class FeedRefreshWorker implements Managed {
feed.setEtagHeader(e.getNewEtagHeader());
}
queues.giveBack(feed);
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
log.debug(e.getClass().getName() + " " + message, e);
@@ -119,7 +101,9 @@ public class FeedRefreshWorker implements Managed {
feed.setMessage(message);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed));
queues.giveBack(feed);
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
@@ -145,22 +129,10 @@ public class FeedRefreshWorker implements Managed {
}
}
private class FeedTask implements Task {
private final FeedRefreshContext context;
public FeedTask(FeedRefreshContext context) {
this.context = context;
}
@Override
public void run() {
update(context);
}
@Override
public boolean isUrgent() {
return context.isUrgent();
}
@Value
public static class FeedRefreshWorkerResult {
Feed feed;
List<FeedEntry> entries;
}
}

View File

@@ -28,6 +28,8 @@ import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
@@ -41,7 +43,6 @@ import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.steadystate.css.parser.CSSOMParser;
import edu.uci.ics.crawler4j.url.URLCanonicalizer;
import lombok.extern.slf4j.Slf4j;
/**
@@ -179,7 +180,10 @@ public class FeedUtils {
if (url == null) {
return null;
}
String normalized = URLCanonicalizer.getCanonicalURL(url);
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}

View File

@@ -1,23 +0,0 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FetchedFeed {
private Feed feed = new Feed();
private List<FeedEntry> entries = new ArrayList<>();
private String title;
private String urlAfterRedirect;
private long fetchDuration;
}

View File

@@ -1,11 +1,9 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@@ -123,7 +121,4 @@ public class Feed extends AbstractModel {
@Temporal(TemporalType.TIMESTAMP)
private Date pushLastPing;
@OneToMany(mappedBy = "feed")
private Set<FeedSubscription> subscriptions;
}

View File

@@ -31,7 +31,7 @@ public class UserSettings extends AbstractModel {
}
public enum ViewMode {
title, cozy, expanded
title, cozy, detailed, expanded
}
@OneToOne(fetch = FetchType.LAZY)

View File

@@ -25,7 +25,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class StartupService implements Managed {
public class DatabaseStartupService implements Managed {
private final SessionFactory sessionFactory;
private final UserDAO userDAO;

View File

@@ -7,18 +7,24 @@ import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryService {
@@ -26,8 +32,45 @@ public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
private final CacheService cache;
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
if (existing != null) {
return false;
}
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink());
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilteringService.FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
return true;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);

View File

@@ -50,6 +50,14 @@ public class FeedService {
return feed;
}
public void save(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
feedDAO.saveOrUpdate(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;

View File

@@ -14,7 +14,7 @@ import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
@@ -35,7 +35,7 @@ public class FeedSubscriptionService {
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedQueues queues;
private final FeedRefreshEngine feedRefreshEngine;
private final CacheService cache;
private final CommaFeedConfiguration config;
@@ -76,7 +76,7 @@ public class FeedSubscriptionService {
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
queues.add(feed, true);
feedRefreshEngine.refreshImmediately(feed);
cache.invalidateUserRootCategory(user);
return sub.getId();
}
@@ -96,7 +96,7 @@ public class FeedSubscriptionService {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
queues.add(feed, true);
feedRefreshEngine.refreshImmediately(feed);
}
}

View File

@@ -1,67 +0,0 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedUpdateService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
if (existing != null) {
return false;
}
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink());
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
return true;
}
}

View File

@@ -17,10 +17,11 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.frontend.resource.PubSubHubbubCallbackREST;
@@ -38,7 +39,8 @@ import lombok.extern.slf4j.Slf4j;
public class PubSubService {
private final CommaFeedConfiguration config;
private final FeedQueues queues;
private final FeedService feedService;
private final SessionFactory sessionFactory;
public void subscribe(Feed feed) {
String hub = feed.getPushHub();
@@ -73,7 +75,7 @@ public class PubSubService {
if (code == 400 && StringUtils.contains(message, pushpressError)) {
String[] tokens = message.split(" ");
feed.setPushTopic(tokens[tokens.length - 1]);
queues.giveBack(feed);
UnitOfWork.run(sessionFactory, () -> feedService.save(feed));
log.debug("handled pushpress subfeed {} : {}", topic, feed.getPushTopic());
} else {
throw new Exception(

View File

@@ -45,9 +45,9 @@ import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher.Favicon;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedFetcher;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.FetchedFeed;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedEntry;
@@ -109,7 +109,7 @@ public class FeedREST {
private final FeedEntryService feedEntryService;
private final FeedSubscriptionService feedSubscriptionService;
private final FeedEntryFilteringService feedEntryFilteringService;
private final FeedQueues queues;
private final FeedRefreshEngine feedRefreshEngine;
private final OPMLImporter opmlImporter;
private final OPMLExporter opmlExporter;
private final CacheService cache;
@@ -244,10 +244,10 @@ public class FeedREST {
url = StringUtils.trimToEmpty(url);
url = prependHttp(url);
try {
FetchedFeed feed = feedFetcher.fetch(url, true, null, null, null, null);
FeedFetcherResult feedFetcherResult = feedFetcher.fetch(url, true, null, null, null, null);
info = new FeedInfo();
info.setUrl(feed.getUrlAfterRedirect());
info.setTitle(feed.getTitle());
info.setUrl(feedFetcherResult.getUrlAfterRedirect());
info.setTitle(feedFetcherResult.getTitle());
} catch (Exception e) {
log.debug(e.getMessage(), e);
@@ -303,7 +303,7 @@ public class FeedREST {
FeedSubscription sub = feedSubscriptionDAO.findById(user, req.getId());
if (sub != null) {
Feed feed = sub.getFeed();
queues.add(feed, true);
feedRefreshEngine.refreshImmediately(feed);
return Response.ok().build();
}
return Response.ok(Status.NOT_FOUND).build();

View File

@@ -26,8 +26,8 @@ import com.codahale.metrics.annotation.Timed;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.feed.FeedParser;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FetchedFeed;
import com.commafeed.backend.feed.FeedParser.FeedParserResult;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.model.Feed;
import com.google.common.base.Preconditions;
@@ -46,7 +46,7 @@ public class PubSubHubbubCallbackREST {
private final FeedDAO feedDAO;
private final FeedParser parser;
private final FeedQueues queues;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
private final MetricRegistry metricRegistry;
@@ -100,8 +100,8 @@ public class PubSubHubbubCallbackREST {
return Response.status(Status.BAD_REQUEST).entity("empty body received").build();
}
FetchedFeed fetchedFeed = parser.parse(null, bytes);
String topic = fetchedFeed.getFeed().getPushTopic();
FeedParserResult feedParserResult = parser.parse(null, bytes);
String topic = feedParserResult.getFeed().getPushTopic();
if (StringUtils.isBlank(topic)) {
return Response.status(Status.BAD_REQUEST).entity("empty topic received").build();
}
@@ -114,7 +114,7 @@ public class PubSubHubbubCallbackREST {
for (Feed feed : feeds) {
log.debug("pushing content to queue for {}", feed.getUrl());
queues.add(feed, false);
feedRefreshEngine.refreshImmediately(feed);
}
metricRegistry.meter(MetricRegistry.name(getClass(), "pushReceived")).mark();

View File

@@ -34,10 +34,12 @@ public class WebSocketSessions {
.flatMap(e -> e.getValue().stream())
.collect(Collectors.toSet());
log.debug("sending '{}' to {} users via websocket", text, userSessions.size());
for (Session userSession : userSessions) {
if (userSession.isOpen()) {
userSession.getAsyncRemote().sendText(text);
if (!userSessions.isEmpty()) {
log.debug("sending '{}' to {} users via websocket", text, userSessions.size());
for (Session userSession : userSessions) {
if (userSession.isOpen()) {
userSession.getAsyncRemote().sendText(text);
}
}
}
}

View File

@@ -1,6 +1,7 @@
package com.commafeed.backend.service;
import org.apache.http.HttpHeaders;
import org.hibernate.SessionFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -15,7 +16,6 @@ import org.mockserver.model.HttpResponse;
import org.mockserver.model.MediaType;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.model.Feed;
@ExtendWith(MockServerExtension.class)
@@ -25,7 +25,10 @@ class PubSubServiceTest {
private CommaFeedConfiguration config;
@Mock
private FeedQueues queues;
private FeedService feedService;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private SessionFactory sessionFactory;
@Mock
private Feed feed;
@@ -40,7 +43,7 @@ class PubSubServiceTest {
this.client = client;
this.client.reset();
this.underTest = new PubSubService(config, queues);
this.underTest = new PubSubService(config, feedService, sessionFactory);
Integer port = client.getPort();
String hubUrl = String.format("http://localhost:%s/hub", port);
@@ -69,7 +72,7 @@ class PubSubServiceTest {
.withMethod("POST")
.withPath("/hub"));
Mockito.verify(feed, Mockito.never()).setPushTopic(Mockito.anyString());
Mockito.verifyNoInteractions(queues);
Mockito.verifyNoInteractions(feedService);
}
@Test
@@ -83,7 +86,7 @@ class PubSubServiceTest {
// Assert
Mockito.verify(feed).setPushTopic(Mockito.anyString());
Mockito.verify(queues).giveBack(feed);
Mockito.verify(feedService).save(feed);
}
@Test
@@ -96,7 +99,7 @@ class PubSubServiceTest {
// Assert
Mockito.verify(feed, Mockito.never()).setPushTopic(Mockito.anyString());
Mockito.verifyNoInteractions(queues);
Mockito.verifyNoInteractions(feedService);
}
}

View File

@@ -1,10 +1,11 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId>
<version>3.0.0</version>
<version>3.1.0</version>
<name>CommaFeed</name>
<packaging>pom</packaging>

32
release.sh Normal file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
# Script to create a new release
# ------------------------------
# exit on error
set -e
# make sure we're on master
BRANCH="$(git rev-parse --abbrev-ref HEAD)"
if [[ "$BRANCH" != "master" ]]; then
echo "You're on branch '$BRANCH', you should be on 'master'."
exit
fi
# make sure README.md has been updated
read -r -p "Has README.md been updated? (Y/n) " CONFIRM
case "$CONFIRM" in
n | N) exit ;;
esac
read -r -p "New version (x.y.z): " VERSION
mvn versions:set -DgenerateBackupPoms=false -DnewVersion="$VERSION"
git add .
git commit -am "release $VERSION"
git tag "$VERSION"
read -r -p "Push master and tag $VERSION? (y/N) " CONFIRM
case "$CONFIRM" in
y | Y) git push --atomic origin master "$VERSION" ;;
esac