Compare commits

..

41 Commits
4.0.0 ... 4.1.0

Author SHA1 Message Date
Athou
874e7dcee6 release 4.1.0 2024-01-12 08:15:40 +01:00
Athou
8297edaf71 redirect to login page instead of welcome page if allowRegistrations is false (#1185) 2024-01-11 16:44:40 +01:00
Athou
9e4e629a1a prevent caching openapi files, so that the documentation is always up to date 2024-01-11 08:01:51 +01:00
Athou
8b86617f18 marking an entry as read/unread now requires to swipe to the left since swiping to the right now opens the mobile menu 2024-01-10 19:57:56 +01:00
Athou
bbda35f868 open sidebar on swipe (#1098) 2024-01-10 19:57:38 +01:00
Athou
df68405fef allow users without email to change their profile (#1184) 2024-01-10 19:28:03 +01:00
Athou
65194d948f ignore vscode files 2024-01-10 11:21:37 +01:00
Athou
d49297216c cleanup 2024-01-10 11:19:47 +01:00
Athou
e3e50f8456 improve artifact upload speed (https://github.com/actions/upload-artifact/issues/199) 2024-01-09 22:08:04 +01:00
Athou
e90b3730ef add JavaTimeModule to RedisCacheService object mapper to be able to serialize java.time.Instant 2024-01-09 22:01:34 +01:00
Athou
7675a24eb6 store only user id in session in order to avoid invalidating all sessions when user model changes 2024-01-09 21:22:20 +01:00
Athou
2bf9186135 only show sidebar resizer when sidebar is actually shown 2024-01-09 16:20:46 +01:00
Athou
d4ea51c145 fix vulnerability 2024-01-09 14:59:33 +01:00
Athou
6e0e99694e use properties file of git-commit-id-maven-plugin so we don't need to filter resources 2024-01-09 14:56:59 +01:00
Athou
9ede8d1c46 remove the Managed interface for classes that are not managed by dropwizard 2024-01-09 14:09:24 +01:00
Athou
fd0425a2be clear all sessions because the session model changed 2024-01-09 11:26:54 +01:00
Athou
2b976cadeb add a memory management section to the readme 2024-01-09 09:39:56 +01:00
Athou
023c27a565 setupListeners is only used for rtk query and we don't use it anymore 2024-01-09 07:24:24 +01:00
Athou
69c9988404 migrate from java.util.Date to java.time 2024-01-08 21:58:40 +01:00
Athou
b1a4debb95 replace toSorted usage with sort (#1183) 2024-01-08 13:48:27 +01:00
Athou
5663d619aa show category hierarchy (#1045) 2024-01-08 13:26:20 +01:00
Athou
2ef9e8d274 add null check 2024-01-07 22:14:00 +01:00
Athou
1292018de0 add setting to delete old entries 2024-01-07 20:49:02 +01:00
Athou
039e91414e prevent demo account from registering custom js code 2024-01-07 17:51:22 +01:00
Athou
662d0f754f avoid flash of light theme when using system color scheme 2024-01-07 17:51:22 +01:00
Athou
7fb7efbdf7 add missing truncate lost in refactoring 2024-01-07 17:51:22 +01:00
Athou
a841c80261 simplify trie building 2024-01-07 17:51:22 +01:00
Athou
da4143fa13 multiple feeds may have the same url hash 2024-01-07 17:51:22 +01:00
Athou
789857b09f compare feed entry content after cleanup because that's what saved in the database 2024-01-07 17:51:22 +01:00
Athou
ed45746f52 extract html cleaning code to its own service 2024-01-07 17:51:22 +01:00
Athou
deb51f2ccc rename FixedSizeSortedSet to FixedSizeSortedList because it's actually a list 2024-01-07 17:51:22 +01:00
Athou
5fec4a4c5f improve lookup by using a set because we only use contains() 2024-01-07 17:51:22 +01:00
Athou
7b335e2fd4 feed refresh engine now uses its own immutable model 2024-01-07 17:51:22 +01:00
Athou
60b6c69020 close the HTTP client after each test to close idle connections (https://github.com/dropwizard/dropwizard/issues/8174) 2024-01-06 08:37:12 +01:00
Athou
08ab32c4c2 we don't need the admin connector for tests 2024-01-05 21:20:56 +01:00
Athou
ff24fe4c7c eslint is already run by vite-plugin-eslint during build 2024-01-05 20:51:41 +01:00
Athou
50c62fb468 remove warning: 'typeParameters' property is deprecated 2024-01-05 20:48:25 +01:00
Athou
201331afc3 update vite to 5.x 2024-01-05 20:38:01 +01:00
Athou
cf3100081e add test for unauthorized websocket usage 2024-01-03 21:08:25 +01:00
Athou
860aab7495 fix typo 2024-01-02 11:11:45 +01:00
Athou
b084c8d108 remove line break 2024-01-02 11:10:33 +01:00
123 changed files with 2129 additions and 1608 deletions

View File

@@ -34,7 +34,7 @@ jobs:
run: mvn --batch-mode --update-snapshots verify run: mvn --batch-mode --update-snapshots verify
- name: Upload JAR - name: Upload JAR
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v4
if: ${{ matrix.java == '17' }} if: ${{ matrix.java == '17' }}
with: with:
name: commafeed.jar name: commafeed.jar

3
.gitignore vendored
View File

@@ -35,5 +35,8 @@ src/main/app/lib
# Sublime # Sublime
*.sublime* *.sublime*
# VSCode
.vscode
# Macs # Macs
*.DS_Store *.DS_Store

View File

@@ -1,5 +1,24 @@
# Changelog # Changelog
## [4.1.0]
- it is now possible to open the sidebar on mobile by swiping to the right (#1098)
- swiping to mark entries as read/unread changed from swipinig right to left because swiping right now opens the sidebar
- the full hierarchy of categories are now displayed in the category dropdown (#1045)
- added a setting `maxEntriesAgeDays` to delete old entries based on their age during database cleanup.
The setting is disabled by default for existing installations, except for the docker image where it is enabled and set
to 365 days
- if user registrations are disabled on your instance which is the default behavior, users are redirected on the login
page instead of the welcome page when not logged in (#1185)
- the sidebar resizer is no longer shown in the middle of the screen on mobile
- when using the system color scheme and the system is using a dark theme, feed entries no longer flicker on load
- the demo account (if enabled) cannot register custom javascript code anymore
- removed the usage of `toSorted` in the client because older browsers do not support it (#1183)
- the openapi documentation is no longer cached by the browser so you always have access to the latest version
- added a memory management section to the readme, reading it is recommended if you are running CommaFeed on a server
with limited memory
- fixed an issue that caused users without an email address set to be unable to edit their profile (#1184)
## [4.0.0] ## [4.0.0]
- migrated from dropwizard 2 to dropwizard 4, Java 17+ is now required - migrated from dropwizard 2 to dropwizard 4, Java 17+ is now required
@@ -11,14 +30,13 @@
- custom JS code is now executed when the app is done loading instead of when the page is loaded - custom JS code is now executed when the app is done loading instead of when the page is loaded
- the favicon is now correctly returned for feeds that return an invalid content type - the favicon is now correctly returned for feeds that return an invalid content type
- the feed refresh engine now uses httpclient5 with connection pooling and no longer creates a new client for each - the feed refresh engine now uses httpclient5 with connection pooling and no longer creates a new client for each
request, request, reducing CPU usage
reducing CPU usage
- updated UI library Mantine to 7.0, improving performance - updated UI library Mantine to 7.0, improving performance
- the h2 embedded database is now compacted on shutdown to reclaim unused space - the h2 embedded database is now compacted on shutdown to reclaim unused space
- the admin connector on port 8084 is now disabled in config.yml.example. Disabling it in your config.yml is - the admin connector on port 8084 is now disabled in config.yml.example. Disabling it in your config.yml is
recommended (see https://github.com/Athou/commafeed/commit/929df60f09cce56020b0962ab111cd8349b271b0) recommended (see https://github.com/Athou/commafeed/commit/929df60f09cce56020b0962ab111cd8349b271b0)
- migrated documentation from swagger 2 to openapi 3 - migrated documentation from swagger 2 to openapi 3
- added a GET method to the fever api to indicate that the endpoint is working correctly when accesed from a browser - added a GET method to the fever api to indicate that the endpoint is working correctly when accessed from a browser
- the websocket connection can now be disabled, the websocket ping interval and the tree reload interval can now be - the websocket connection can now be disabled, the websocket ping interval and the tree reload interval can now be
configured (see config.yml.example) configured (see config.yml.example)
- the websocket connection now works correctly when the context root of the application is not "/" - the websocket connection now works correctly when the context root of the application is not "/"

View File

@@ -54,6 +54,29 @@ user is `admin` and the default password is `admin`.
The server will listen on http://localhost:8082. The default The server will listen on http://localhost:8082. The default
user is `admin` and the default password is `admin`. user is `admin` and the default password is `admin`.
### Memory management
The Java Virtual Machine (JVM) is rather greedy by default and will not release unused memory to the
operating system. This is because acquiring memory from the operating system is a relatively expensive operation.
However, this can be problematic on systems with limited memory.
#### Hard limit
The JVM can be configured to use a maximum amount of memory with the `-Xmx` parameter.
For example, to limit the JVM to 256MB of memory, use `-Xmx256m`.
#### Dynamic sizing
The JVM can be configured to release unused memory to the operating system with the following parameters:
-Xms20m -XX:+UseG1GC -XX:-ShrinkHeapInSteps -XX:G1PeriodicGCInterval=10000 -XX:-G1PeriodicGCInvokesConcurrent -XX:MinHeapFreeRatio=5 -XX:MaxHeapFreeRatio=10
This is how the Docker image is configured.
See [here](https://docs.oracle.com/en/java/javase/17/gctuning/garbage-first-g1-garbage-collector1.html)
and [here](https://docs.oracle.com/en/java/javase/17/gctuning/factors-affecting-garbage-collection-performance.html) for
more
information.
## Translation ## Translation
Files for internationalization are Files for internationalization are

File diff suppressed because it is too large Load Diff

View File

@@ -16,9 +16,9 @@
"dependencies": { "dependencies": {
"@emotion/react": "^11.11.3", "@emotion/react": "^11.11.3",
"@fontsource/open-sans": "^5.0.20", "@fontsource/open-sans": "^5.0.20",
"@lingui/core": "^4.6.0", "@lingui/core": "^4.7.0",
"@lingui/macro": "^4.6.0", "@lingui/macro": "^4.7.0",
"@lingui/react": "^4.6.0", "@lingui/react": "^4.7.0",
"@mantine/core": "^7.3.2", "@mantine/core": "^7.3.2",
"@mantine/form": "^7.3.2", "@mantine/form": "^7.3.2",
"@mantine/hooks": "^7.3.2", "@mantine/hooks": "^7.3.2",
@@ -52,8 +52,8 @@
"websocket-heartbeat-js": "^1.1.3" "websocket-heartbeat-js": "^1.1.3"
}, },
"devDependencies": { "devDependencies": {
"@lingui/cli": "^4.6.0", "@lingui/cli": "^4.7.0",
"@lingui/vite-plugin": "^4.6.0", "@lingui/vite-plugin": "^4.7.0",
"@types/mousetrap": "^1.6.15", "@types/mousetrap": "^1.6.15",
"@types/react": "^18.2.46", "@types/react": "^18.2.46",
"@types/react-dom": "^18.2.18", "@types/react-dom": "^18.2.18",
@@ -76,10 +76,10 @@
"prettier": "^3.1.1", "prettier": "^3.1.1",
"rollup-plugin-visualizer": "^5.12.0", "rollup-plugin-visualizer": "^5.12.0",
"typescript": "^5.3.3", "typescript": "^5.3.3",
"vite": "^4.5.1", "vite": "^5.0.11",
"vite-plugin-eslint": "^1.8.1", "vite-plugin-eslint": "^1.8.1",
"vite-tsconfig-paths": "^4.2.3", "vite-tsconfig-paths": "^4.2.3",
"vitest": "^0.34.6", "vitest": "^1.1.3",
"vitest-mock-extended": "^1.3.1" "vitest-mock-extended": "^1.3.1"
} }
} }

View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.commafeed</groupId> <groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId> <artifactId>commafeed</artifactId>
<version>4.0.0</version> <version>4.1.0</version>
</parent> </parent>
<artifactId>commafeed-client</artifactId> <artifactId>commafeed-client</artifactId>
<name>CommaFeed Client</name> <name>CommaFeed Client</name>
@@ -39,16 +39,6 @@
<arguments>ci</arguments> <arguments>ci</arguments>
</configuration> </configuration>
</execution> </execution>
<execution>
<id>npm run eslint</id>
<goals>
<goal>npm</goal>
</goals>
<phase>compile</phase>
<configuration>
<arguments>run eslint</arguments>
</configuration>
</execution>
<execution> <execution>
<id>npm run test</id> <id>npm run test</id>
<goals> <goals>

View File

@@ -31,11 +31,12 @@ const axiosInstance = axios.create({ baseURL: "./rest", withCredentials: true })
axiosInstance.interceptors.response.use( axiosInstance.interceptors.response.use(
response => response, response => response,
error => { error => {
const { status, data } = error.response
if ( if (
(error.response.status === 401 && error.response.data === "Credentials are required to access this resource.") || (status === 401 && data?.message === "Credentials are required to access this resource.") ||
(error.response.status === 403 && error.response.data === "You don't have the required role to access this resource.") (status === 403 && data?.message === "You don't have the required role to access this resource.")
) { ) {
window.location.hash = "/welcome" window.location.hash = data?.allowRegistrations ? "/welcome" : "/login"
} }
throw error throw error
} }

View File

@@ -1,5 +1,4 @@
import { configureStore } from "@reduxjs/toolkit" import { configureStore } from "@reduxjs/toolkit"
import { setupListeners } from "@reduxjs/toolkit/query"
import { entriesSlice } from "app/entries/slice" import { entriesSlice } from "app/entries/slice"
import { redirectSlice } from "app/redirect/slice" import { redirectSlice } from "app/redirect/slice"
import { serverSlice } from "app/server/slice" import { serverSlice } from "app/server/slice"
@@ -17,8 +16,6 @@ export const reducers = {
export const store = configureStore({ reducer: reducers }) export const store = configureStore({ reducer: reducers })
setupListeners(store.dispatch)
export type RootState = ReturnType<typeof store.getState> export type RootState = ReturnType<typeof store.getState>
export type AppDispatch = typeof store.dispatch export type AppDispatch = typeof store.dispatch

View File

@@ -110,7 +110,7 @@ export function KeyboardShortcutsHelp() {
<Table.Td> <Table.Td>
<Kbd>M</Kbd> <Kbd>M</Kbd>
<span>, </span> <span>, </span>
<Trans>Swipe header to the right</Trans> <Trans>Swipe header to the left</Trans>
</Table.Td> </Table.Td>
</Table.Tr> </Table.Tr>
<Table.Tr> <Table.Tr>

View File

@@ -91,7 +91,7 @@ export function FeedEntries() {
) )
} }
const swipedRight = async (entry: ExpendableEntry) => await dispatch(markEntry({ entry, read: !entry.read })) const swipedLeft = async (entry: ExpendableEntry) => await dispatch(markEntry({ entry, read: !entry.read }))
// close context menu on scroll // close context menu on scroll
useEffect(() => { useEffect(() => {
@@ -320,7 +320,7 @@ export function FeedEntries() {
onHeaderClick={event => headerClicked(entry, event)} onHeaderClick={event => headerClicked(entry, event)}
onHeaderRightClick={event => headerRightClicked(entry, event)} onHeaderRightClick={event => headerRightClicked(entry, event)}
onBodyClick={() => bodyClicked(entry)} onBodyClick={() => bodyClicked(entry)}
onSwipedRight={async () => await swipedRight(entry)} onSwipedLeft={async () => await swipedLeft(entry)}
/> />
</div> </div>
))} ))}

View File

@@ -21,7 +21,7 @@ interface FeedEntryProps {
onHeaderClick: (e: React.MouseEvent) => void onHeaderClick: (e: React.MouseEvent) => void
onHeaderRightClick: (e: React.MouseEvent) => void onHeaderRightClick: (e: React.MouseEvent) => void
onBodyClick: (e: React.MouseEvent) => void onBodyClick: (e: React.MouseEvent) => void
onSwipedRight: () => void onSwipedLeft: () => void
} }
const useStyles = tss const useStyles = tss
@@ -111,7 +111,7 @@ export function FeedEntry(props: FeedEntryProps) {
}) })
const swipeHandlers = useSwipeable({ const swipeHandlers = useSwipeable({
onSwipedRight: props.onSwipedRight, onSwipedLeft: props.onSwipedLeft,
}) })
let paddingX: MantineSpacing = "xs" let paddingX: MantineSpacing = "xs"

View File

@@ -3,6 +3,7 @@ import { Select, type SelectProps } from "@mantine/core"
import { type ComboboxItem } from "@mantine/core/lib/components/Combobox/Combobox.types" import { type ComboboxItem } from "@mantine/core/lib/components/Combobox/Combobox.types"
import { Constants } from "app/constants" import { Constants } from "app/constants"
import { useAppSelector } from "app/store" import { useAppSelector } from "app/store"
import { type Category } from "app/types"
import { flattenCategoryTree } from "app/utils" import { flattenCategoryTree } from "app/utils"
type CategorySelectProps = Partial<SelectProps> & { type CategorySelectProps = Partial<SelectProps> & {
@@ -13,14 +14,32 @@ type CategorySelectProps = Partial<SelectProps> & {
export function CategorySelect(props: CategorySelectProps) { export function CategorySelect(props: CategorySelectProps) {
const rootCategory = useAppSelector(state => state.tree.rootCategory) const rootCategory = useAppSelector(state => state.tree.rootCategory)
const categories = rootCategory && flattenCategoryTree(rootCategory) const categories = rootCategory && flattenCategoryTree(rootCategory)
const categoriesById = categories?.reduce((map, c) => {
map.set(c.id, c)
return map
}, new Map<string, Category>())
const categoryLabel = (cat: Category) => {
let label = cat.name
while (cat.parentId) {
const parent = categoriesById?.get(cat.parentId)
if (!parent) {
break
}
label = `${parent.name}${label}`
cat = parent
}
return label
}
const selectData: ComboboxItem[] | undefined = categories const selectData: ComboboxItem[] | undefined = categories
?.filter(c => c.id !== Constants.categories.all.id) ?.filter(c => c.id !== Constants.categories.all.id)
.filter(c => !props.withoutCategoryIds?.includes(c.id)) .filter(c => !props.withoutCategoryIds?.includes(c.id))
.sort((c1, c2) => c1.name.localeCompare(c2.name))
.map(c => ({ .map(c => ({
label: c.parentName ? t`${c.name} (in ${c.parentName})` : c.name, label: categoryLabel(c),
value: c.id, value: c.id,
})) }))
.sort((c1, c2) => c1.label.localeCompare(c2.label))
if (props.withAll) { if (props.withAll) {
selectData?.unshift({ selectData?.unshift({
label: t`All`, label: t`All`,

View File

@@ -16,13 +16,13 @@ export function TreeSearch(props: TreeSearchProps) {
const dispatch = useAppDispatch() const dispatch = useAppDispatch()
const actions: SpotlightActionData[] = props.feeds const actions: SpotlightActionData[] = props.feeds
.toSorted((f1, f2) => f1.name.localeCompare(f2.name))
.map(f => ({ .map(f => ({
id: `${f.id}`, id: `${f.id}`,
label: f.name, label: f.name,
leftSection: <FeedFavicon url={f.iconUrl} />, leftSection: <FeedFavicon url={f.iconUrl} />,
onClick: async () => await dispatch(redirectToFeed(f.id)), onClick: async () => await dispatch(redirectToFeed(f.id)),
})) }))
.sort((f1, f2) => f1.label.localeCompare(f2.label))
const searchIcon = <TbSearch size={18} /> const searchIcon = <TbSearch size={18} />
const rightSection = ( const rightSection = (

View File

@@ -1,4 +1,20 @@
import { useComputedColorScheme } from "@mantine/core"
// the color scheme to use to render components // the color scheme to use to render components
export const useColorScheme = () => useComputedColorScheme("light") import { useMantineColorScheme } from "@mantine/core"
import { useMediaQuery } from "@mantine/hooks"
export const useColorScheme = () => {
const systemColorScheme = useMediaQuery(
"(prefers-color-scheme: dark)",
// passing undefined will use window.matchMedia(query) as default value
undefined,
{
// get initial value synchronously and not in useEffect to avoid flash of light theme
getInitialValueInEffect: false,
}
)
? "dark"
: "light"
const { colorScheme } = useMantineColorScheme()
return colorScheme === "auto" ? systemColorScheme : colorScheme
}

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "النجاح" msgstr "النجاح"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Éxit" msgstr "Éxit"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Úspěch" msgstr "Úspěch"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Llwyddiant" msgstr "Llwyddiant"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Succes" msgstr "Succes"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "CommaFeed ist ein Open Source Projekt. Der Quellcode wird auf auf </0><1>GitHub</1> gehostet." msgstr "CommaFeed ist ein Open Source Projekt. Der Quellcode wird auf auf </0><1>GitHub</1> gehostet."
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Erfolg" msgstr "Erfolg"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr "{0} (in {1})"
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgstr "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
@@ -823,8 +819,8 @@ msgid "Success"
msgstr "Success" msgstr "Success"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "Swipe header to the right" msgstr "Swipe header to the left"
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx
msgid "Switch to dark theme" msgid "Switch to dark theme"

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Éxito" msgstr "Éxito"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "موفقیت" msgstr "موفقیت"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Onnistui" msgstr "Onnistui"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr "{0} (sur {1})"
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "<0>CommaFeed est un projet open-source. Les sources sont hébergées sur </0><1>GitHub</1>." msgstr "<0>CommaFeed est un projet open-source. Les sources sont hébergées sur </0><1>GitHub</1>."
@@ -823,8 +819,8 @@ msgid "Success"
msgstr "Succès" msgstr "Succès"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "Faire glisser le titre vers la droite" msgstr "Faire glisser le titre vers la gauche"
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx
msgid "Switch to dark theme" msgid "Switch to dark theme"

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Éxito" msgstr "Éxito"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Siker" msgstr "Siker"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Sukses" msgstr "Sukses"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Successo" msgstr "Successo"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "成功" msgstr "成功"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "성공" msgstr "성공"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Kejayaan" msgstr "Kejayaan"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Suksess" msgstr "Suksess"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Succes" msgstr "Succes"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Suksess" msgstr "Suksess"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Sukces" msgstr "Sukces"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Sucesso" msgstr "Sucesso"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Успех" msgstr "Успех"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Úspech" msgstr "Úspech"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Framgång" msgstr "Framgång"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr "{0} ({1} içinde)"
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "<0>CommaFeed açık kaynak kodlu bir proje. Kaynak kodları </0><1>GitHub</1>'da." msgstr "<0>CommaFeed açık kaynak kodlu bir proje. Kaynak kodları </0><1>GitHub</1>'da."
@@ -823,8 +819,8 @@ msgid "Success"
msgstr "Başarı" msgstr "Başarı"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "Başlığı sağa kaydır" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx
msgid "Switch to dark theme" msgid "Switch to dark theme"

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n" "Language-Team: \n"
"Plural-Forms: \n" "Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx #: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>." msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "" msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "成功" msgstr "成功"
#: src/components/KeyboardShortcutsHelp.tsx #: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right" msgid "Swipe header to the left"
msgstr "" msgstr ""
#: src/pages/WelcomePage.tsx #: src/pages/WelcomePage.tsx

View File

@@ -19,6 +19,7 @@ import { type ReactNode, Suspense, useEffect } from "react"
import Draggable from "react-draggable" import Draggable from "react-draggable"
import { TbMenu2, TbPlus, TbX } from "react-icons/tb" import { TbMenu2, TbPlus, TbX } from "react-icons/tb"
import { Outlet } from "react-router-dom" import { Outlet } from "react-router-dom"
import { useSwipeable } from "react-swipeable"
import { tss } from "tss" import { tss } from "tss"
import useLocalStorage from "use-local-storage" import useLocalStorage from "use-local-storage"
@@ -111,81 +112,94 @@ export default function Layout(props: LayoutProps) {
</ActionIcon> </ActionIcon>
) )
const swipeHandlers = useSwipeable({
onSwiping: e => {
const threshold = document.documentElement.clientWidth / 6
if (e.absX > threshold) {
dispatch(setMobileMenuOpen(e.dir === "Right"))
}
},
})
if (loading) return <LoadingPage /> if (loading) return <LoadingPage />
return ( return (
<AppShell <Box {...swipeHandlers}>
header={{ height: Constants.layout.headerHeight }} <AppShell
navbar={{ header={{ height: Constants.layout.headerHeight }}
width: sidebarWidth, navbar={{
breakpoint: Constants.layout.mobileBreakpoint, width: sidebarWidth,
collapsed: { mobile: !mobileMenuOpen, desktop: !props.sidebarVisible }, breakpoint: Constants.layout.mobileBreakpoint,
}} collapsed: { mobile: !mobileMenuOpen, desktop: !props.sidebarVisible },
padding={{ base: 6, [Constants.layout.mobileBreakpointName]: "md" }} }}
> padding={{ base: 6, [Constants.layout.mobileBreakpointName]: "md" }}
<AppShell.Header id="header"> >
<OnMobile> <AppShell.Header id="header">
{mobileMenuOpen && ( <OnMobile>
<Group justify="space-between" p="md"> {mobileMenuOpen && (
<Box>{burger}</Box> <Group justify="space-between" p="md">
<Box> <Box>{burger}</Box>
<LogoAndTitle /> <Box>
</Box> <LogoAndTitle />
<Box>{addButton}</Box> </Box>
</Group> <Box>{addButton}</Box>
)} </Group>
{!mobileMenuOpen && ( )}
{!mobileMenuOpen && (
<Group p="md">
<Box>{burger}</Box>
<Box style={{ flexGrow: 1 }}>{props.header}</Box>
</Group>
)}
</OnMobile>
<OnDesktop>
<Group p="md"> <Group p="md">
<Box>{burger}</Box> <Group justify="space-between" style={{ width: sidebarWidth - 16 }}>
<Box>
<LogoAndTitle />
</Box>
<Box>{addButton}</Box>
</Group>
<Box style={{ flexGrow: 1 }}>{props.header}</Box> <Box style={{ flexGrow: 1 }}>{props.header}</Box>
</Group> </Group>
)} </OnDesktop>
</OnMobile> </AppShell.Header>
<AppShell.Navbar id="sidebar" p={sidebarPadding}>
<AppShell.Section grow component={ScrollArea} mx="-sm" px="sm">
<Box className={classes.sidebarContent}>{props.sidebar}</Box>
</AppShell.Section>
</AppShell.Navbar>
<OnDesktop> <OnDesktop>
<Group p="md"> <Draggable
<Group justify="space-between" style={{ width: sidebarWidth - 16 }}> axis="x"
<Box> defaultPosition={{
<LogoAndTitle /> x: sidebarWidth,
</Box> y: Constants.layout.headerHeight,
<Box>{addButton}</Box> }}
</Group> bounds={{
<Box style={{ flexGrow: 1 }}>{props.header}</Box> left: 120,
</Group> right: 1000,
}}
grid={[30, 30]}
onDrag={(_e, data) => setSidebarWidth(data.x)}
>
<Box
style={{
position: "fixed",
height: "100%",
width: "10px",
cursor: "ew-resize",
}}
></Box>
</Draggable>
</OnDesktop> </OnDesktop>
</AppShell.Header>
<AppShell.Navbar id="sidebar" p={sidebarPadding}>
<AppShell.Section grow component={ScrollArea} mx="-sm" px="sm">
<Box className={classes.sidebarContent}>{props.sidebar}</Box>
</AppShell.Section>
</AppShell.Navbar>
<Draggable
axis="x"
defaultPosition={{
x: sidebarWidth,
y: Constants.layout.headerHeight,
}}
bounds={{
left: 120,
right: 1000,
}}
grid={[30, 30]}
onDrag={(_e, data) => setSidebarWidth(data.x)}
>
<Box
style={{
position: "fixed",
height: "100%",
width: "10px",
cursor: "ew-resize",
}}
></Box>
</Draggable>
<AppShell.Main id="content"> <AppShell.Main id="content">
<Suspense fallback={<Loader />}> <Suspense fallback={<Loader />}>
<AnnouncementDialog /> <AnnouncementDialog />
<Outlet /> <Outlet />
</Suspense> </Suspense>
</AppShell.Main> </AppShell.Main>
</AppShell> </AppShell>
</Box>
) )
} }

View File

@@ -6,7 +6,7 @@ import eslint from "vite-plugin-eslint"
import tsconfigPaths from "vite-tsconfig-paths" import tsconfigPaths from "vite-tsconfig-paths"
// https://vitejs.dev/config/ // https://vitejs.dev/config/
export default defineConfig({ export default defineConfig(env => ({
plugins: [ plugins: [
react({ react({
babel: { babel: {
@@ -15,7 +15,8 @@ export default defineConfig({
}, },
}), }),
lingui(), lingui(),
eslint(), // https://github.com/vitest-dev/vitest/issues/4055#issuecomment-1732994672
env.mode !== "test" && eslint(),
tsconfigPaths(), tsconfigPaths(),
visualizer(), visualizer(),
], ],
@@ -32,7 +33,7 @@ export default defineConfig({
}, },
}, },
build: { build: {
chunkSizeWarningLimit: 3000, chunkSizeWarningLimit: 3500,
rollupOptions: { rollupOptions: {
output: { output: {
manualChunks: id => { manualChunks: id => {
@@ -44,4 +45,4 @@ export default defineConfig({
}, },
}, },
}, },
}) }))

View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable # entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500 maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 365
# limit the number of feeds a user can subscribe to, 0 to disable # limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0 maxFeedsPerUser: 0

View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable # entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500 maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 365
# limit the number of feeds a user can subscribe to, 0 to disable # limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0 maxFeedsPerUser: 0

View File

@@ -6,7 +6,7 @@
<parent> <parent>
<groupId>com.commafeed</groupId> <groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId> <artifactId>commafeed</artifactId>
<version>4.0.0</version> <version>4.1.0</version>
</parent> </parent>
<artifactId>commafeed-server</artifactId> <artifactId>commafeed-server</artifactId>
<name>CommaFeed Server</name> <name>CommaFeed Server</name>
@@ -31,12 +31,6 @@
<build> <build>
<finalName>commafeed</finalName> <finalName>commafeed</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins> <plugins>
<plugin> <plugin>
@@ -69,7 +63,8 @@
</execution> </execution>
</executions> </executions>
<configuration> <configuration>
<generateGitPropertiesFile>false</generateGitPropertiesFile> <generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
<failOnNoGitDirectory>false</failOnNoGitDirectory> <failOnNoGitDirectory>false</failOnNoGitDirectory>
<failOnUnableToExtractRepoInfo>false</failOnUnableToExtractRepoInfo> <failOnUnableToExtractRepoInfo>false</failOnUnableToExtractRepoInfo>
</configuration> </configuration>
@@ -216,7 +211,7 @@
<dependency> <dependency>
<groupId>com.commafeed</groupId> <groupId>com.commafeed</groupId>
<artifactId>commafeed-client</artifactId> <artifactId>commafeed-client</artifactId>
<version>4.0.0</version> <version>4.1.0</version>
</dependency> </dependency>
<dependency> <dependency>

View File

@@ -1,7 +1,7 @@
package com.commafeed; package com.commafeed;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.time.Instant;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
@@ -10,6 +10,7 @@ import java.util.concurrent.TimeUnit;
import org.hibernate.cfg.AvailableSettings; import org.hibernate.cfg.AvailableSettings;
import com.codahale.metrics.json.MetricsModule; import com.codahale.metrics.json.MetricsModule;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.feed.FeedRefreshEngine; import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.model.AbstractModel; import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
@@ -42,8 +43,10 @@ import com.commafeed.frontend.servlet.RobotsTxtDisallowAllServlet;
import com.commafeed.frontend.session.SessionHelperFactoryProvider; import com.commafeed.frontend.session.SessionHelperFactoryProvider;
import com.commafeed.frontend.ws.WebSocketConfigurator; import com.commafeed.frontend.ws.WebSocketConfigurator;
import com.commafeed.frontend.ws.WebSocketEndpoint; import com.commafeed.frontend.ws.WebSocketEndpoint;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.Injector; import com.google.inject.Injector;
import com.google.inject.Key; import com.google.inject.Key;
@@ -76,7 +79,7 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
public static final String USERNAME_ADMIN = "admin"; public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo"; public static final String USERNAME_DEMO = "demo";
public static final Date STARTUP_TIME = new Date(); public static final Instant STARTUP_TIME = Instant.now();
private HibernateBundle<CommaFeedConfiguration> hibernateBundle; private HibernateBundle<CommaFeedConfiguration> hibernateBundle;
private WebsocketBundle<CommaFeedConfiguration> websocketBundle; private WebsocketBundle<CommaFeedConfiguration> websocketBundle;
@@ -89,8 +92,7 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
@Override @Override
public void initialize(Bootstrap<CommaFeedConfiguration> bootstrap) { public void initialize(Bootstrap<CommaFeedConfiguration> bootstrap) {
configureEnvironmentSubstitutor(bootstrap); configureEnvironmentSubstitutor(bootstrap);
configureObjectMapper(bootstrap.getObjectMapper());
bootstrap.getObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
bootstrap.addBundle(websocketBundle = new WebsocketBundle<>()); bootstrap.addBundle(websocketBundle = new WebsocketBundle<>());
bootstrap.addBundle(hibernateBundle = new HibernateBundle<>(AbstractModel.class, Feed.class, FeedCategory.class, FeedEntry.class, bootstrap.addBundle(hibernateBundle = new HibernateBundle<>(AbstractModel.class, Feed.class, FeedCategory.class, FeedEntry.class,
@@ -134,6 +136,15 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
bootstrap.setConfigurationSourceProvider(buildEnvironmentSubstitutor(bootstrap)); bootstrap.setConfigurationSourceProvider(buildEnvironmentSubstitutor(bootstrap));
} }
private static void configureObjectMapper(ObjectMapper objectMapper) {
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
private static EnvironmentSubstitutor buildEnvironmentSubstitutor(Bootstrap<CommaFeedConfiguration> bootstrap) { private static EnvironmentSubstitutor buildEnvironmentSubstitutor(Bootstrap<CommaFeedConfiguration> bootstrap) {
// enable config.yml string substitution // enable config.yml string substitution
// e.g. having a custom config.yml file with app.session.path=${SOME_ENV_VAR} will substitute SOME_ENV_VAR // e.g. having a custom config.yml file with app.session.path=${SOME_ENV_VAR} will substitute SOME_ENV_VAR
@@ -156,7 +167,9 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
environment.servlets().setSessionHandler(config.getSessionHandlerFactory().build(config.getDataSourceFactory())); environment.servlets().setSessionHandler(config.getSessionHandlerFactory().build(config.getDataSourceFactory()));
// support for "@SecurityCheck User user" injection // support for "@SecurityCheck User user" injection
environment.jersey().register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserService.class))); environment.jersey()
.register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserDAO.class),
injector.getInstance(UserService.class), config));
// support for "@Context SessionHelper sessionHelper" injection // support for "@Context SessionHelper sessionHelper" injection
environment.jersey().register(new SessionHelperFactoryProvider.Binder()); environment.jersey().register(new SessionHelperFactoryProvider.Binder());
@@ -208,6 +221,12 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
environment.servlets() environment.servlets()
.addFilter("index-cache-busting-filter", new CacheBustingFilter()) .addFilter("index-cache-busting-filter", new CacheBustingFilter())
.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/"); .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/");
// prevent caching openapi files, so that the documentation is always up to date
environment.servlets()
.addFilter("openapi-cache-busting-filter", new CacheBustingFilter())
.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/openapi.json", "/openapi.yaml");
// prevent caching REST resources, except for favicons // prevent caching REST resources, except for favicons
environment.servlets().addFilter("rest-cache-busting-filter", new CacheBustingFilter() { environment.servlets().addFilter("rest-cache-busting-filter", new CacheBustingFilter() {
@Override @Override

View File

@@ -1,9 +1,10 @@
package com.commafeed; package com.commafeed;
import java.util.Date; import java.io.IOException;
import java.util.ResourceBundle; import java.io.InputStream;
import java.time.Instant;
import org.apache.commons.lang3.time.DateUtils; import java.time.temporal.ChronoUnit;
import java.util.Properties;
import com.commafeed.backend.cache.RedisPoolFactory; import com.commafeed.backend.cache.RedisPoolFactory;
import com.commafeed.frontend.session.SessionHandlerFactory; import com.commafeed.frontend.session.SessionHandlerFactory;
@@ -54,10 +55,17 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
private final String gitCommit; private final String gitCommit;
public CommaFeedConfiguration() { public CommaFeedConfiguration() {
ResourceBundle bundle = ResourceBundle.getBundle("application"); Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
this.version = bundle.getString("version"); this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = bundle.getString("git.commit"); this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
} }
@Override @Override
@@ -146,6 +154,11 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
@Valid @Valid
private Integer maxFeedCapacity; private Integer maxFeedCapacity;
@NotNull
@Min(0)
@Valid
private Integer maxEntriesAgeDays = 0;
@NotNull @NotNull
@Valid @Valid
private Integer maxFeedsPerUser = 0; private Integer maxFeedsPerUser = 0;
@@ -170,9 +183,8 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
private Duration treeReloadInterval = Duration.seconds(30); private Duration treeReloadInterval = Duration.seconds(30);
public Date getUnreadThreshold() { public Instant getUnreadThreshold() {
int keepStatusDays = getKeepStatusDays(); return getKeepStatusDays() > 0 ? Instant.now().minus(getKeepStatusDays(), ChronoUnit.DAYS) : null;
return keepStatusDays > 0 ? DateUtils.addDays(new Date(), -1 * keepStatusDays) : null;
} }
} }

View File

@@ -19,6 +19,7 @@ import com.commafeed.backend.favicon.DefaultFaviconFetcher;
import com.commafeed.backend.favicon.FacebookFaviconFetcher; import com.commafeed.backend.favicon.FacebookFaviconFetcher;
import com.commafeed.backend.favicon.YoutubeFaviconFetcher; import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
import com.commafeed.backend.task.DemoAccountCleanupTask; import com.commafeed.backend.task.DemoAccountCleanupTask;
import com.commafeed.backend.task.EntriesExceedingFeedCapacityCleanupTask;
import com.commafeed.backend.task.OldEntriesCleanupTask; import com.commafeed.backend.task.OldEntriesCleanupTask;
import com.commafeed.backend.task.OldStatusesCleanupTask; import com.commafeed.backend.task.OldStatusesCleanupTask;
import com.commafeed.backend.task.OrphanedContentsCleanupTask; import com.commafeed.backend.task.OrphanedContentsCleanupTask;
@@ -66,6 +67,7 @@ public class CommaFeedModule extends AbstractModule {
Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class); Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
taskMultibinder.addBinding().to(OldStatusesCleanupTask.class); taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
taskMultibinder.addBinding().to(EntriesExceedingFeedCapacityCleanupTask.class);
taskMultibinder.addBinding().to(OldEntriesCleanupTask.class); taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class); taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class); taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);

View File

@@ -10,14 +10,14 @@ import java.util.List;
* *
* *
*/ */
public class FixedSizeSortedSet<E> { public class FixedSizeSortedList<E> {
private final List<E> inner; private final List<E> inner;
private final Comparator<? super E> comparator; private final Comparator<? super E> comparator;
private final int capacity; private final int capacity;
public FixedSizeSortedSet(int capacity, Comparator<? super E> comparator) { public FixedSizeSortedList(int capacity, Comparator<? super E> comparator) {
this.inner = new ArrayList<>(Math.max(0, capacity)); this.inner = new ArrayList<>(Math.max(0, capacity));
this.capacity = capacity < 0 ? Integer.MAX_VALUE : capacity; this.capacity = capacity < 0 ? Integer.MAX_VALUE : capacity;
this.comparator = comparator; this.comparator = comparator;

View File

@@ -31,7 +31,6 @@ import com.commafeed.CommaFeedConfiguration;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.common.net.HttpHeaders; import com.google.common.net.HttpHeaders;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.Getter; import lombok.Getter;
@@ -44,7 +43,7 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils;
* *
*/ */
@Singleton @Singleton
public class HttpGetter implements Managed { public class HttpGetter {
private final CloseableHttpClient client; private final CloseableHttpClient client;
@@ -154,11 +153,6 @@ public class HttpGetter implements Managed {
.build(); .build();
} }
@Override
public void stop() throws Exception {
client.close();
}
@Getter @Getter
public static class NotModifiedException extends Exception { public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;

View File

@@ -1,11 +1,12 @@
package com.commafeed.backend.cache; package com.commafeed.backend.cache;
import java.util.List; import java.util.List;
import java.util.Set;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category; import com.commafeed.frontend.model.Category;
@@ -14,12 +15,12 @@ import com.commafeed.frontend.model.UnreadCount;
public abstract class CacheService { public abstract class CacheService {
// feed entries for faster refresh // feed entries for faster refresh
public abstract List<String> getLastEntries(Feed feed); public abstract Set<String> getLastEntries(Feed feed);
public abstract void setLastEntries(Feed feed, List<String> entries); public abstract void setLastEntries(Feed feed, List<String> entries);
public String buildUniqueEntryKey(Feed feed, FeedEntry entry) { public String buildUniqueEntryKey(Entry entry) {
return DigestUtils.sha1Hex(entry.getGuid() + entry.getUrl()); return DigestUtils.sha1Hex(entry.guid() + entry.url());
} }
// user categories // user categories

View File

@@ -2,6 +2,7 @@ package com.commafeed.backend.cache;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Set;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
@@ -12,8 +13,8 @@ import com.commafeed.frontend.model.UnreadCount;
public class NoopCacheService extends CacheService { public class NoopCacheService extends CacheService {
@Override @Override
public List<String> getLastEntries(Feed feed) { public Set<String> getLastEntries(Feed feed) {
return Collections.emptyList(); return Collections.emptySet();
} }
@Override @Override

View File

@@ -1,6 +1,5 @@
package com.commafeed.backend.cache; package com.commafeed.backend.cache;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -13,6 +12,7 @@ import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount; import com.commafeed.frontend.model.UnreadCount;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -24,19 +24,16 @@ import redis.clients.jedis.Pipeline;
@RequiredArgsConstructor @RequiredArgsConstructor
public class RedisCacheService extends CacheService { public class RedisCacheService extends CacheService {
private static final ObjectMapper MAPPER = new ObjectMapper(); private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
private final JedisPool pool; private final JedisPool pool;
@Override @Override
public List<String> getLastEntries(Feed feed) { public Set<String> getLastEntries(Feed feed) {
List<String> list = new ArrayList<>();
try (Jedis jedis = pool.getResource()) { try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed); String key = buildRedisEntryKey(feed);
Set<String> members = jedis.smembers(key); return jedis.smembers(key);
list.addAll(members);
} }
return list;
} }
@Override @Override

View File

@@ -1,16 +1,14 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.Date; import java.time.Instant;
import java.util.List; import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.hibernate.SessionFactory; import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed; import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription; import com.commafeed.backend.model.QFeedSubscription;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPAExpressions; import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
@@ -28,8 +26,8 @@ public class FeedDAO extends GenericDAO<Feed> {
super(sessionFactory); super(sessionFactory);
} }
public List<Feed> findNextUpdatable(int count, Date lastLoginThreshold) { public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(feed).where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(new Date()))); JPAQuery<Feed> query = query().selectFrom(feed).where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) { if (lastLoginThreshold != null) {
query.where(JPAExpressions.selectOne() query.where(JPAExpressions.selectOne()
.from(subscription) .from(subscription)
@@ -41,17 +39,18 @@ public class FeedDAO extends GenericDAO<Feed> {
return query.orderBy(feed.disabledUntil.asc()).limit(count).fetch(); return query.orderBy(feed.disabledUntil.asc()).limit(count).fetch();
} }
public void setDisabledUntil(List<Long> feedIds, Date date) { public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(feed).set(feed.disabledUntil, date).where(feed.id.in(feedIds)).execute(); updateQuery(feed).set(feed.disabledUntil, date).where(feed.id.in(feedIds)).execute();
} }
public Feed findByUrl(String normalizedUrl) { public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
List<Feed> feeds = query().selectFrom(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).fetch(); return query().selectFrom(feed)
Feed feed = Iterables.getFirst(feeds, null); .where(feed.normalizedUrlHash.eq(normalizedUrlHash))
if (feed != null && StringUtils.equals(normalizedUrl, feed.getNormalizedUrl())) { .fetch()
return feed; .stream()
} .filter(f -> StringUtils.equals(normalizedUrl, f.getNormalizedUrl()))
return null; .findFirst()
.orElse(null);
} }
public List<Feed> findWithoutSubscriptions(int max) { public List<Feed> findWithoutSubscriptions(int max) {

View File

@@ -1,8 +1,8 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List; import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import org.hibernate.SessionFactory; import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
@@ -26,12 +26,8 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
super(sessionFactory); super(sessionFactory);
} }
public Long findExisting(String guid, Feed feed) { public Long findExisting(String guidHash, Feed feed) {
return query().select(entry.id) return query().select(entry.id).from(entry).where(entry.guidHash.eq(guidHash), entry.feed.eq(feed)).limit(1).fetchOne();
.from(entry)
.where(entry.guidHash.eq(DigestUtils.sha1Hex(guid)), entry.feed.eq(feed))
.limit(1)
.fetchOne();
} }
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) { public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
@@ -50,6 +46,17 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return delete(list); return delete(list);
} }
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.updated.lt(olderThan)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) { public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch(); List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list); return delete(list);

View File

@@ -1,8 +1,8 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date;
import java.util.List; import java.util.List;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
@@ -10,7 +10,7 @@ import org.apache.commons.lang3.builder.CompareToBuilder;
import org.hibernate.SessionFactory; import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.FixedSizeSortedSet; import com.commafeed.backend.FixedSizeSortedList;
import com.commafeed.backend.feed.FeedEntryKeyword; import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode; import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
@@ -73,8 +73,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) { private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) { if (status == null) {
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold(); Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
boolean read = unreadThreshold != null && entry.getUpdated().before(unreadThreshold); boolean read = unreadThreshold != null && entry.getUpdated().isBefore(unreadThreshold);
status = new FeedEntryStatus(user, sub, entry); status = new FeedEntryStatus(user, sub, entry);
status.setRead(read); status.setRead(read);
status.setMarkable(!read); status.setMarkable(!read);
@@ -90,7 +90,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
return status; return status;
} }
public List<FeedEntryStatus> findStarred(User user, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent) { public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue()); JPAQuery<FeedEntryStatus> query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue());
if (newerThan != null) { if (newerThan != null) {
query.where(status.entryInserted.gt(newerThan)); query.where(status.entryInserted.gt(newerThan));
@@ -114,7 +115,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
} }
private JPAQuery<FeedEntry> buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List<FeedEntryKeyword> keywords, private JPAQuery<FeedEntry> buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List<FeedEntryKeyword> keywords,
Date newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag, Long minEntryId, Long maxEntryId) { Instant newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag, Long minEntryId,
Long maxEntryId) {
JPAQuery<FeedEntry> query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed())); JPAQuery<FeedEntry> query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed()));
@@ -139,7 +141,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
or.or(status.read.isFalse()); or.or(status.read.isFalse());
query.where(or); query.where(or);
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold(); Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
if (unreadThreshold != null) { if (unreadThreshold != null) {
query.where(entry.updated.goe(unreadThreshold)); query.where(entry.updated.goe(unreadThreshold));
} }
@@ -193,22 +195,22 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
} }
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly, public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent, List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
boolean onlyIds, String tag, Long minEntryId, Long maxEntryId) { boolean onlyIds, String tag, Long minEntryId, Long maxEntryId) {
int capacity = offset + limit; int capacity = offset + limit;
Comparator<FeedEntryStatus> comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC; Comparator<FeedEntryStatus> comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC;
FixedSizeSortedSet<FeedEntryStatus> set = new FixedSizeSortedSet<>(capacity, comparator); FixedSizeSortedList<FeedEntryStatus> fssl = new FixedSizeSortedList<>(capacity, comparator);
for (FeedSubscription sub : subs) { for (FeedSubscription sub : subs) {
FeedEntryStatus last = (order != null && set.isFull()) ? set.last() : null; FeedEntryStatus last = (order != null && fssl.isFull()) ? fssl.last() : null;
JPAQuery<FeedEntry> query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag, minEntryId, JPAQuery<FeedEntry> query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag, minEntryId,
maxEntryId); maxEntryId);
List<Tuple> tuples = query.select(entry.id, entry.updated, status.id, entry.content.title).fetch(); List<Tuple> tuples = query.select(entry.id, entry.updated, status.id, entry.content.title).fetch();
for (Tuple tuple : tuples) { for (Tuple tuple : tuples) {
Long id = tuple.get(entry.id); Long id = tuple.get(entry.id);
Date updated = tuple.get(entry.updated); Instant updated = tuple.get(entry.updated);
Long statusId = tuple.get(status.id); Long statusId = tuple.get(status.id);
FeedEntryContent content = new FeedEntryContent(); FeedEntryContent content = new FeedEntryContent();
@@ -225,11 +227,11 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
status.setEntry(entry); status.setEntry(entry);
status.setSubscription(sub); status.setSubscription(sub);
set.add(status); fssl.add(status);
} }
} }
List<FeedEntryStatus> placeholders = set.asList(); List<FeedEntryStatus> placeholders = fssl.asList();
int size = placeholders.size(); int size = placeholders.size();
if (size < offset) { if (size < offset) {
return new ArrayList<>(); return new ArrayList<>();
@@ -260,7 +262,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
List<Tuple> tuples = query.select(entry.count(), entry.updated.max()).fetch(); List<Tuple> tuples = query.select(entry.count(), entry.updated.max()).fetch();
for (Tuple tuple : tuples) { for (Tuple tuple : tuples) {
Long count = tuple.get(entry.count()); Long count = tuple.get(entry.count());
Date updated = tuple.get(entry.updated.max()); Instant updated = tuple.get(entry.updated.max());
uc = new UnreadCount(subscription.getId(), count == null ? 0 : count, updated); uc = new UnreadCount(subscription.getId(), count == null ? 0 : count, updated);
} }
return uc; return uc;
@@ -276,7 +278,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
return results; return results;
} }
public long deleteOldStatuses(Date olderThan, int limit) { public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(status.id) List<Long> ids = query().select(status.id)
.from(status) .from(status)
.where(status.entryInserted.lt(olderThan), status.starred.isFalse()) .where(status.entryInserted.lt(olderThan), status.starred.isFalse())

View File

@@ -1,8 +1,7 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.time.Instant;
import java.util.List;
import java.util.Set; import java.util.Set;
import org.apache.commons.codec.binary.StringUtils; import org.apache.commons.codec.binary.StringUtils;
@@ -11,16 +10,14 @@ import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException; import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedParser.FeedParserResult; import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.urlprovider.FeedURLProvider; import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException; import com.rometools.rome.io.FeedException;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
/** /**
@@ -35,8 +32,8 @@ public class FeedFetcher {
private final HttpGetter getter; private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders; private final Set<FeedURLProvider> urlProviders;
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate, public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
String lastContentHash) throws FeedException, IOException, NotModifiedException { Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl); log.debug("Fetching feed {}", feedUrl);
int timeout = 20000; int timeout = 20000;
@@ -79,20 +76,15 @@ public class FeedFetcher {
etagHeaderValueChanged ? result.getETag() : null); etagHeaderValueChanged ? result.getETag() : null);
} }
if (lastPublishedDate != null && parserResult.getFeed().getLastPublishedDate() != null if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
&& lastPublishedDate.getTime() == parserResult.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl); log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified", throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null, lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null); etagHeaderValueChanged ? result.getETag() : null);
} }
Feed feed = parserResult.getFeed(); return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
feed.setLastModifiedHeader(result.getLastModifiedSince()); result.getDuration());
feed.setEtagHeader(FeedUtils.truncate(result.getETag(), 255));
feed.setLastContentHash(hash);
return new FeedFetcherResult(parserResult.getFeed(), parserResult.getEntries(), parserResult.getTitle(),
result.getUrlAfterRedirect(), result.getDuration());
} }
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) { private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
@@ -106,13 +98,8 @@ public class FeedFetcher {
return null; return null;
} }
@Value public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
public static class FeedFetcherResult { String contentHash, long fetchDuration) {
Feed feed;
List<FeedEntry> entries;
String title;
String urlAfterRedirect;
long fetchDuration;
} }
} }

View File

@@ -1,263 +0,0 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.Value;
/**
* Parses raw xml as a Feed object
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedParser {
private static final String ATOM_10_URI = "http://www.w3.org/2005/Atom";
private static final Namespace ATOM_10_NS = Namespace.getNamespace(ATOM_10_URI);
private static final Date START = new Date(86400000);
private static final Date END = new Date(1000L * Integer.MAX_VALUE - 86400000);
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedException {
try {
Charset encoding = FeedUtils.guessEncoding(xml);
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
}
xmlString = FeedUtils.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed rss = new SyndFeedInput().build(source);
handleForeignMarkup(rss);
String title = rss.getTitle();
Feed feed = new Feed();
feed.setUrl(feedUrl);
feed.setLink(rss.getLink());
List<FeedEntry> entries = new ArrayList<>();
for (SyndEntry item : rss.getEntries()) {
FeedEntry entry = new FeedEntry();
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
entry.setGuid(FeedUtils.truncate(guid, 2048));
entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feedUrl), 2048));
// if link is empty but guid is used as url
if (StringUtils.isBlank(entry.getUrl()) && StringUtils.startsWith(entry.getGuid(), "http")) {
entry.setUrl(entry.getGuid());
}
FeedEntryContent content = new FeedEntryContent();
content.setContent(getContent(item));
content.setCategories(FeedUtils
.truncate(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")), 4096));
content.setTitle(getTitle(item));
content.setAuthor(StringUtils.trimToNull(item.getAuthor()));
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure != null) {
content.setEnclosureUrl(FeedUtils.truncate(enclosure.getUrl(), 2048));
content.setEnclosureType(enclosure.getType());
}
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module != null) {
Media media = getMedia(module);
if (media != null) {
content.setMediaDescription(media.getDescription());
content.setMediaThumbnailUrl(FeedUtils.truncate(media.getThumbnailUrl(), 2048));
content.setMediaThumbnailWidth(media.getThumbnailWidth());
content.setMediaThumbnailHeight(media.getThumbnailHeight());
}
}
entry.setContent(content);
entries.add(entry);
}
Date lastEntryDate = null;
Date publishedDate = validateDate(rss.getPublishedDate(), false);
if (!entries.isEmpty()) {
List<Long> sortedTimestamps = FeedUtils.getSortedTimestamps(entries);
Long timestamp = sortedTimestamps.get(0);
lastEntryDate = new Date(timestamp);
publishedDate = (publishedDate == null || publishedDate.before(lastEntryDate)) ? lastEntryDate : publishedDate;
}
feed.setLastPublishedDate(publishedDate);
feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
feed.setLastEntryDate(lastEntryDate);
return new FeedParserResult(feed, entries, title);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private Date getEntryUpdateDate(SyndEntry item) {
Date date = item.getUpdatedDate();
if (date == null) {
date = item.getPublishedDate();
}
if (date == null) {
date = new Date();
}
return date;
}
private Date validateDate(Date date, boolean nullToNow) {
Date now = new Date();
if (date == null) {
return nullToNow ? now : null;
}
if (date.before(START) || date.after(END)) {
return now;
}
if (date.after(now)) {
return now;
}
return date;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media getMedia(MediaEntryModule module) {
Media media = getMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = getMedia(group.getMetadata());
}
return media;
}
private Media getMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
Media media = new Media();
media.setDescription(metadata.getDescription());
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
media.setThumbnailWidth(thumbnail.getWidth());
media.setThumbnailHeight(thumbnail.getHeight());
if (thumbnail.getUrl() != null) {
media.setThumbnailUrl(thumbnail.getUrl().toString());
}
}
if (media.isEmpty()) {
return null;
}
return media;
}
@Data
private static class Media {
private String description;
private String thumbnailUrl;
private Integer thumbnailWidth;
private Integer thumbnailHeight;
public boolean isEmpty() {
return description == null && thumbnailUrl == null;
}
}
@Value
public static class FeedParserResult {
Feed feed;
List<FeedEntry> entries;
String title;
}
}

View File

@@ -1,6 +1,7 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.util.Date; import java.time.Duration;
import java.time.Instant;
import java.util.List; import java.util.List;
import java.util.concurrent.BlockingDeque; import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
@@ -11,8 +12,6 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.time.DateUtils;
import com.codahale.metrics.Gauge; import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
@@ -156,7 +155,7 @@ public class FeedRefreshEngine implements Managed {
private void processFeedAsync(Feed feed) { private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor) CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.getFeed(), r.getEntries()), databaseUpdaterExecutor) .thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> { .whenComplete((data, ex) -> {
if (ex != null) { if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex); log.error("error while processing feed {}", feed.getUrl(), ex);
@@ -166,12 +165,12 @@ public class FeedRefreshEngine implements Managed {
private List<Feed> getNextUpdatableFeeds(int max) { private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> { return unitOfWork.call(() -> {
Date lastLoginThreshold = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad()) Instant lastLoginThreshold = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad())
? DateUtils.addDays(new Date(), -30) ? Instant.now().minus(Duration.ofDays(30))
: null; : null;
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold); List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable() // update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Date nextUpdateDate = DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes()); Instant nextUpdateDate = Instant.now().plus(Duration.ofMinutes(config.getApplicationSettings().getRefreshIntervalMinutes()));
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate); feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds; return feeds;
}); });

View File

@@ -1,11 +1,10 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.util.Date; import java.time.Duration;
import java.time.Instant;
import org.apache.commons.lang3.time.DateUtils; import java.time.temporal.ChronoUnit;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
@@ -22,62 +21,59 @@ public class FeedRefreshIntervalCalculator {
this.refreshIntervalMinutes = config.getApplicationSettings().getRefreshIntervalMinutes(); this.refreshIntervalMinutes = config.getApplicationSettings().getRefreshIntervalMinutes();
} }
public Date onFetchSuccess(Feed feed) { public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval) {
Date defaultRefreshInterval = getDefaultRefreshInterval(); Instant defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(feed, defaultRefreshInterval) : defaultRefreshInterval; return heavyLoad ? computeRefreshIntervalForHeavyLoad(publishedDate, averageEntryInterval, defaultRefreshInterval)
: defaultRefreshInterval;
} }
public Date onFeedNotModified(Feed feed) { public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
Date defaultRefreshInterval = getDefaultRefreshInterval(); return onFetchSuccess(publishedDate, averageEntryInterval);
return heavyLoad ? computeRefreshIntervalForHeavyLoad(feed, defaultRefreshInterval) : defaultRefreshInterval;
} }
public Date onFetchError(Feed feed) { public Instant onFetchError(int errorCount) {
int errorCount = feed.getErrorCount();
int retriesBeforeDisable = 3; int retriesBeforeDisable = 3;
if (errorCount < retriesBeforeDisable || !heavyLoad) { if (errorCount < retriesBeforeDisable || !heavyLoad) {
return getDefaultRefreshInterval(); return getDefaultRefreshInterval();
} }
int disabledHours = Math.min(24 * 7, errorCount - retriesBeforeDisable + 1); int disabledHours = Math.min(24 * 7, errorCount - retriesBeforeDisable + 1);
return DateUtils.addHours(new Date(), disabledHours); return Instant.now().plus(Duration.ofHours(disabledHours));
} }
private Date getDefaultRefreshInterval() { private Instant getDefaultRefreshInterval() {
return DateUtils.addMinutes(new Date(), refreshIntervalMinutes); return Instant.now().plus(Duration.ofMinutes(refreshIntervalMinutes));
} }
private Date computeRefreshIntervalForHeavyLoad(Feed feed, Date defaultRefreshInterval) { private Instant computeRefreshIntervalForHeavyLoad(Instant publishedDate, Long averageEntryInterval, Instant defaultRefreshInterval) {
Date now = new Date(); Instant now = Instant.now();
Date publishedDate = feed.getLastEntryDate();
Long averageEntryInterval = feed.getAverageEntryInterval();
if (publishedDate == null) { if (publishedDate == null) {
// feed with no entries, recheck in 24 hours // feed with no entries, recheck in 24 hours
return DateUtils.addHours(now, 24); return now.plus(Duration.ofHours(24));
} else if (publishedDate.before(DateUtils.addMonths(now, -1))) { } else if (ChronoUnit.DAYS.between(publishedDate, now) >= 30) {
// older than a month, recheck in 24 hours // older than a month, recheck in 24 hours
return DateUtils.addHours(now, 24); return now.plus(Duration.ofHours(24));
} else if (publishedDate.before(DateUtils.addDays(now, -14))) { } else if (ChronoUnit.DAYS.between(publishedDate, now) >= 14) {
// older than two weeks, recheck in 12 hours // older than two weeks, recheck in 12 hours
return DateUtils.addHours(now, 12); return now.plus(Duration.ofHours(12));
} else if (publishedDate.before(DateUtils.addDays(now, -7))) { } else if (ChronoUnit.DAYS.between(publishedDate, now) >= 7) {
// older than a week, recheck in 6 hours // older than a week, recheck in 6 hours
return DateUtils.addHours(now, 6); return now.plus(Duration.ofHours(6));
} else if (averageEntryInterval != null) { } else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor // use average time between entries to decide when to refresh next, divided by factor
int factor = 2; int factor = 2;
// not more than 6 hours // not more than 6 hours
long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor); long date = Math.min(now.plus(Duration.ofHours(6)).toEpochMilli(), now.toEpochMilli() + averageEntryInterval / factor);
// not less than default refresh interval // not less than default refresh interval
date = Math.max(defaultRefreshInterval.getTime(), date); date = Math.max(defaultRefreshInterval.toEpochMilli(), date);
return new Date(date); return Instant.ofEpochMilli(date);
} else { } else {
// unknown case, recheck in 24 hours // unknown case, recheck in 24 hours
return DateUtils.addHours(now, 24); return now.plus(Duration.ofHours(24));
} }
} }

View File

@@ -1,10 +1,11 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
@@ -16,9 +17,9 @@ import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.cache.CacheService; import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryService; import com.commafeed.backend.service.FeedEntryService;
@@ -27,7 +28,6 @@ import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions; import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped; import com.google.common.util.concurrent.Striped;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@@ -38,7 +38,7 @@ import lombok.extern.slf4j.Slf4j;
*/ */
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedRefreshUpdater implements Managed { public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final FeedService feedService; private final FeedService feedService;
@@ -72,7 +72,7 @@ public class FeedRefreshUpdater implements Managed {
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted")); entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
} }
private AddEntryResult addEntry(final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) { private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false; boolean processed = false;
boolean inserted = false; boolean inserted = false;
@@ -82,8 +82,8 @@ public class FeedRefreshUpdater implements Managed {
// lock on content, make sure we are not updating the same entry // lock on content, make sure we are not updating the same entry
// twice at the same time // twice at the same time
FeedEntryContent content = entry.getContent(); Content content = entry.content();
String key2 = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent() + content.getTitle())); String key2 = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator(); Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next(); Lock lock1 = iterator.next();
@@ -116,19 +116,19 @@ public class FeedRefreshUpdater implements Managed {
return new AddEntryResult(processed, inserted); return new AddEntryResult(processed, inserted);
} }
public boolean update(Feed feed, List<FeedEntry> entries) { public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true; boolean processed = true;
boolean insertedAtLeastOneEntry = false; boolean insertedAtLeastOneEntry = false;
if (!entries.isEmpty()) { if (!entries.isEmpty()) {
List<String> lastEntries = cache.getLastEntries(feed); Set<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>(); List<String> currentEntries = new ArrayList<>();
List<FeedSubscription> subscriptions = null; List<FeedSubscription> subscriptions = null;
for (FeedEntry entry : entries) { for (Entry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(feed, entry); String cacheKey = cache.buildUniqueEntryKey(entry);
if (!lastEntries.contains(cacheKey)) { if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl()); log.debug("cache miss for {}", entry.url());
if (subscriptions == null) { if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed)); subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
} }
@@ -138,7 +138,7 @@ public class FeedRefreshUpdater implements Managed {
entryCacheMiss.mark(); entryCacheMiss.mark();
} else { } else {
log.debug("cache hit for {}", entry.getUrl()); log.debug("cache hit for {}", entry.url());
entryCacheHit.mark(); entryCacheHit.mark();
} }
@@ -160,7 +160,7 @@ public class FeedRefreshUpdater implements Managed {
if (!processed) { if (!processed) {
// requeue asap // requeue asap
feed.setDisabledUntil(new Date(0)); feed.setDisabledUntil(Instant.EPOCH);
} }
if (insertedAtLeastOneEntry) { if (insertedAtLeastOneEntry) {

View File

@@ -1,5 +1,7 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@@ -11,16 +13,15 @@ import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException; import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult; import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.Value;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
/** /**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database ({@link FeedRefreshUpdater} does that) * Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/ */
@Slf4j @Slf4j
@Singleton @Singleton
@@ -44,32 +45,41 @@ public class FeedRefreshWorker {
public FeedRefreshWorkerResult update(Feed feed) { public FeedRefreshWorkerResult update(Feed feed) {
try { try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl()); String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult feedFetcherResult = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(), FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash()); feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown // stops here if NotModifiedException or any other exception is thrown
List<FeedEntry> entries = feedFetcherResult.getEntries();
List<Entry> entries = result.feed().entries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) { if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList(); entries = entries.stream().limit(maxFeedCapacity).toList();
} }
String urlAfterRedirect = feedFetcherResult.getUrlAfterRedirect(); Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxEntriesAgeDays > 0) {
Instant threshold = Instant.now().minus(Duration.ofDays(maxEntriesAgeDays));
entries = entries.stream().filter(entry -> entry.updated().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) { if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null; urlAfterRedirect = null;
} }
feed.setUrlAfterRedirect(urlAfterRedirect); feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(feedFetcherResult.getFeed().getLink()); feed.setLink(result.feed().link());
feed.setLastModifiedHeader(feedFetcherResult.getFeed().getLastModifiedHeader()); feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(feedFetcherResult.getFeed().getEtagHeader()); feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(feedFetcherResult.getFeed().getLastContentHash()); feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(feedFetcherResult.getFeed().getLastPublishedDate()); feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(feedFetcherResult.getFeed().getAverageEntryInterval()); feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(feedFetcherResult.getFeed().getLastEntryDate()); feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0); feed.setErrorCount(0);
feed.setMessage(null); feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(feedFetcherResult.getFeed())); feed.setDisabledUntil(
refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), result.feed().averageEntryInterval()));
return new FeedRefreshWorkerResult(feed, entries); return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) { } catch (NotModifiedException e) {
@@ -77,7 +87,7 @@ public class FeedRefreshWorker {
feed.setErrorCount(0); feed.setErrorCount(0);
feed.setMessage(e.getMessage()); feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed)); feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) { if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader()); feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
@@ -93,7 +103,7 @@ public class FeedRefreshWorker {
feed.setErrorCount(feed.getErrorCount() + 1); feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage()); feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed)); feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList()); return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally { } finally {
@@ -101,10 +111,7 @@ public class FeedRefreshWorker {
} }
} }
@Value public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
public static class FeedRefreshWorkerResult {
Feed feed;
List<FeedEntry> entries;
} }
} }

View File

@@ -2,20 +2,12 @@ package com.commafeed.backend.feed;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.ahocorasick.trie.Trie.TrieBuilder;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@@ -29,8 +21,6 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry; import com.commafeed.frontend.model.Entry;
import com.google.gwt.i18n.client.HasDirection.Direction; import com.google.gwt.i18n.client.HasDirection.Direction;
import com.google.gwt.i18n.shared.BidiUtils; import com.google.gwt.i18n.shared.BidiUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -50,70 +40,6 @@ public class FeedUtils {
return string; return string;
} }
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public static Charset guessEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Detect encoding by analyzing characters in the array
*/
public static Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
public static String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
TrieBuilder builder = Trie.builder();
builder.ignoreOverlaps();
for (String key : HtmlEntities.HTML_ENTITIES) {
builder.addKeyword(key);
}
Trie trie = builder.build();
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public static boolean isHttp(String url) { public static boolean isHttp(String url) {
return url.startsWith("http://"); return url.startsWith("http://");
} }
@@ -122,6 +48,10 @@ public class FeedUtils {
return url.startsWith("https://"); return url.startsWith("https://");
} }
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/** /**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates * Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/ */
@@ -163,25 +93,6 @@ public class FeedUtils {
return normalized; return normalized;
} }
/**
* Extract the declared encoding from the xml
*/
public static String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
public static boolean isRTL(FeedEntry entry) { public static boolean isRTL(FeedEntry entry) {
String text = entry.getContent().getContent(); String text = entry.getContent().getContent();
@@ -202,52 +113,6 @@ public class FeedUtils {
return direction == Direction.RTL; return direction == Direction.RTL;
} }
public static String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
public static Long averageTimeBetweenEntries(List<FeedEntry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
List<Long> timestamps = getSortedTimestamps(entries);
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < timestamps.size() - 1; i++) {
long diff = Math.abs(timestamps.get(i) - timestamps.get(i + 1));
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static List<Long> getSortedTimestamps(List<FeedEntry> entries) {
return entries.stream().map(t -> t.getUpdated().getTime()).sorted(Collections.reverseOrder()).toList();
}
public static String removeTrailingSlash(String url) { public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) { if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1); url = url.substring(0, url.length() - 1);
@@ -256,8 +121,8 @@ public class FeedUtils {
} }
/** /**
* *
* @param url * @param relativeUrl
* the url of the entry * the url of the entry
* @param feedLink * @param feedLink
* the url of the feed as described in the feed * the url of the feed as described in the feed
@@ -265,32 +130,18 @@ public class FeedUtils {
* the url of the feed that we used to fetch the feed * the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry * @return an absolute url pointing to the entry
*/ */
public static String toAbsoluteUrl(String url, String feedLink, String feedUrl) { public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
url = StringUtils.trimToNull(StringUtils.normalizeSpace(url)); String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (url == null || url.startsWith("http")) {
return url;
}
String baseUrl = (feedLink == null || isRelative(feedLink)) ? feedUrl : feedLink;
if (baseUrl == null) { if (baseUrl == null) {
return url; return null;
} }
String result;
try { try {
result = new URL(new URL(baseUrl), url).toString(); return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
log.debug("could not parse url : " + e.getMessage(), e); log.debug("could not parse url : " + e.getMessage(), e);
result = url; return null;
} }
return result;
}
public static boolean isRelative(final String url) {
// the regex means "start with 'scheme://'"
return url.startsWith("/") || url.startsWith("#") || !url.matches("^\\w+\\:\\/\\/.*");
} }
public static String getFaviconUrl(FeedSubscription subscription) { public static String getFaviconUrl(FeedSubscription subscription) {

View File

@@ -0,0 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import jakarta.inject.Singleton;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

View File

@@ -0,0 +1,63 @@
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
import jakarta.inject.Singleton;
@Singleton
class FeedCleaner {
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618/1885506
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
}

View File

@@ -0,0 +1,271 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::updated).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant updated = buildEntryUpdateDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, updated, content));
}
entries.sort(Comparator.comparing(Entry::updated).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryUpdateDate(SyndEntry item) {
Date date = item.getUpdatedDate();
if (date == null) {
date = item.getPublishedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).updated().toEpochMilli() - entries.get(i + 1).updated().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
}

View File

@@ -0,0 +1,20 @@
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant updated, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}

View File

@@ -1,10 +1,13 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed.parser;
import java.util.Collections; import java.util.Collections;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
public class HtmlEntities { import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP; public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES; public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES; public static final String[] NUMERIC_ENTITIES;

View File

@@ -1,12 +1,10 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.util.Date; import java.time.Instant;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -44,20 +42,20 @@ public class Feed extends AbstractModel {
/** /**
* Last time we tried to fetch the feed * Last time we tried to fetch the feed
*/ */
@Temporal(TemporalType.TIMESTAMP) @Column
private Date lastUpdated; private Instant lastUpdated;
/** /**
* Last publishedDate value in the feed * Last publishedDate value in the feed
*/ */
@Temporal(TemporalType.TIMESTAMP) @Column
private Date lastPublishedDate; private Instant lastPublishedDate;
/** /**
* date of the last entry of the feed * date of the last entry of the feed
*/ */
@Temporal(TemporalType.TIMESTAMP) @Column
private Date lastEntryDate; private Instant lastEntryDate;
/** /**
* error message while retrieving the feed * error message while retrieving the feed
@@ -73,8 +71,8 @@ public class Feed extends AbstractModel {
/** /**
* feed refresh is disabled until this date * feed refresh is disabled until this date
*/ */
@Temporal(TemporalType.TIMESTAMP) @Column
private Date disabledUntil; private Instant disabledUntil;
/** /**
* http header returned by the feed * http header returned by the feed

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.util.Date; import java.time.Instant;
import java.util.Set; import java.util.Set;
import jakarta.persistence.CascadeType; import jakarta.persistence.CascadeType;
@@ -11,8 +11,6 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany; import jakarta.persistence.OneToMany;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -39,11 +37,11 @@ public class FeedEntry extends AbstractModel {
@Column(length = 2048) @Column(length = 2048)
private String url; private String url;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date inserted; private Instant inserted;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date updated; private Instant updated;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE) @OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses; private Set<FeedEntryStatus> statuses;

View File

@@ -69,13 +69,13 @@ public class FeedEntryContent extends AbstractModel {
return new EqualsBuilder().append(title, c.title) return new EqualsBuilder().append(title, c.title)
.append(content, c.content) .append(content, c.content)
.append(author, c.author) .append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl) .append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType) .append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription) .append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl) .append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth) .append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight) .append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.append(categories, c.categories)
.build(); .build();
} }

View File

@@ -1,7 +1,7 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;
import jakarta.persistence.Column; import jakarta.persistence.Column;
@@ -10,8 +10,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn; import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import jakarta.persistence.Transient; import jakarta.persistence.Transient;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -49,11 +47,11 @@ public class FeedEntryStatus extends AbstractModel {
@JoinColumn(nullable = false) @JoinColumn(nullable = false)
private User user; private User user;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date entryInserted; private Instant entryInserted;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date entryUpdated; private Instant entryUpdated;
public FeedEntryStatus() { public FeedEntryStatus() {

View File

@@ -1,12 +1,10 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.util.Date; import java.time.Instant;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -35,15 +33,15 @@ public class User extends AbstractModel {
@Column(nullable = false) @Column(nullable = false)
private boolean disabled; private boolean disabled;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date lastLogin; private Instant lastLogin;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date created; private Instant created;
@Column(length = 40) @Column(length = 40)
private String recoverPasswordToken; private String recoverPasswordToken;
@Temporal(TemporalType.TIMESTAMP) @Column
private Date recoverPasswordTokenDate; private Instant recoverPasswordTokenDate;
} }

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.util.Date; import java.time.Instant;
import java.util.List; import java.util.List;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
@@ -83,6 +83,7 @@ public class DatabaseCleaningService {
} }
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0; long total = 0;
while (true) { while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize)); List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
@@ -105,7 +106,20 @@ public class DatabaseCleaningService {
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total); log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
} }
public void cleanStatusesOlderThan(final Date olderThan) { public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.info("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses"); log.info("cleaning old read statuses");
long total = 0; long total = 0;
long deleted; long deleted;

View File

@@ -0,0 +1,174 @@
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser();
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}

View File

@@ -1,206 +1,69 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.commafeed.backend.dao.FeedEntryContentDAO; import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent; import com.commafeed.backend.model.FeedEntryContent;
import com.steadystate.css.parser.CSSOMParser;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject })) @RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Slf4j
@Singleton @Singleton
public class FeedEntryContentService { public class FeedEntryContentService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/** /**
* this is NOT thread-safe * this is NOT thread-safe
*/ */
public FeedEntryContent findOrCreate(FeedEntryContent content, String baseUrl) { public FeedEntryContent findOrCreate(Content content, String baseUrl) {
content.setAuthor(FeedUtils.truncate(handleContent(content.getAuthor(), baseUrl, true), 128)); FeedEntryContent entryContent = buildContent(content, baseUrl);
content.setTitle(FeedUtils.truncate(handleContent(content.getTitle(), baseUrl, true), 2048)); Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
content.setContent(handleContent(content.getContent(), baseUrl, false)); .stream()
content.setMediaDescription(handleContent(content.getMediaDescription(), baseUrl, false)); .filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
String contentHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent())); private FeedEntryContent buildContent(Content content, String baseUrl) {
content.setContentHash(contentHash); FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
String titleHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getTitle())); Enclosure enclosure = content.enclosure();
content.setTitleHash(titleHash); if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
List<FeedEntryContent> existing = feedEntryContentDAO.findExisting(contentHash, titleHash); entryContent.setEnclosureType(enclosure.type());
Optional<FeedEntryContent> equivalentContent = existing.stream().filter(content::equivalentTo).findFirst();
if (equivalentContent.isPresent()) {
return equivalentContent.get();
} }
feedEntryContentDAO.saveOrUpdate(content); Media media = content.media();
return content; if (media != null) {
} entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
private static Safelist buildWhiteList() { entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
Safelist whitelist = new Safelist(); entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String handleContent(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
} }
return content;
return entryContent;
} }
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser();
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
} }

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.util.Date; import java.time.Instant;
import java.util.List; import java.util.List;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
@@ -10,9 +10,10 @@ import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword; import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus; import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
@@ -37,30 +38,27 @@ public class FeedEntryService {
/** /**
* this is NOT thread-safe * this is NOT thread-safe
*/ */
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) { public boolean addEntry(Feed feed, Entry entry, List<FeedSubscription> subscriptions) {
String guid = FeedUtils.truncate(entry.guid(), 2048);
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed); String guidHash = DigestUtils.sha1Hex(entry.guid());
Long existing = feedEntryDAO.findExisting(guidHash, feed);
if (existing != null) { if (existing != null) {
return false; return false;
} }
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink()); FeedEntry feedEntry = buildEntry(feed, entry, guid, guidHash);
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid())); feedEntryDAO.saveOrUpdate(feedEntry);
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
// if filter does not match the entry, mark it as read // if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) { for (FeedSubscription sub : subscriptions) {
boolean matches = true; boolean matches = true;
try { try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry); matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), feedEntry);
} catch (FeedEntryFilteringService.FeedEntryFilterException e) { } catch (FeedEntryFilteringService.FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e); log.error("could not evaluate filter {}", sub.getFilter(), e);
} }
if (!matches) { if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry); FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, feedEntry);
status.setRead(true); status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status); feedEntryStatusDAO.saveOrUpdate(status);
} }
@@ -69,8 +67,20 @@ public class FeedEntryService {
return true; return true;
} }
public void markEntry(User user, Long entryId, boolean read) { private FeedEntry buildEntry(Feed feed, Entry e, String guid, String guidHash) {
FeedEntry entry = new FeedEntry();
entry.setGuid(guid);
entry.setGuidHash(guidHash);
entry.setUrl(FeedUtils.truncate(e.url(), 2048));
entry.setUpdated(e.updated());
entry.setInserted(Instant.now());
entry.setFeed(feed);
entry.setContent(feedEntryContentService.findOrCreate(e.content(), feed.getLink()));
return entry;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId); FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) { if (entry == null) {
return; return;
@@ -107,7 +117,7 @@ public class FeedEntryService {
feedEntryStatusDAO.saveOrUpdate(status); feedEntryStatusDAO.saveOrUpdate(status);
} }
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Date olderThan, Date insertedBefore, public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) { List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null, List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, false, null, null, null); false, false, null, null, null);
@@ -116,18 +126,18 @@ public class FeedEntryService {
cache.invalidateUserRootCategory(user); cache.invalidateUserRootCategory(user);
} }
public void markStarredEntries(User user, Date olderThan, Date insertedBefore) { public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false); List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore); markList(statuses, olderThan, insertedBefore);
} }
private void markList(List<FeedEntryStatus> statuses, Date olderThan, Date insertedBefore) { private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(s -> { List<FeedEntryStatus> statusesToMark = statuses.stream().filter(s -> {
Date entryDate = s.getEntry().getUpdated(); Instant entryDate = s.getEntry().getUpdated();
return olderThan == null || entryDate == null || entryDate.before(olderThan); return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> { }).filter(s -> {
Date insertedDate = s.getEntry().getInserted(); Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.before(insertedBefore); return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList(); }).toList();
statusesToMark.forEach(s -> s.setRead(true)); statusesToMark.forEach(s -> s.setRead(true));

View File

@@ -1,7 +1,7 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.time.Instant;
import java.util.Set; import java.util.Set;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
@@ -37,14 +37,15 @@ public class FeedService {
} }
public synchronized Feed findOrCreate(String url) { public synchronized Feed findOrCreate(String url) {
String normalized = FeedUtils.normalizeURL(url); String normalizedUrl = FeedUtils.normalizeURL(url);
Feed feed = feedDAO.findByUrl(normalized); String normalizedUrlHash = DigestUtils.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) { if (feed == null) {
feed = new Feed(); feed = new Feed();
feed.setUrl(url); feed.setUrl(url);
feed.setNormalizedUrl(normalized); feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized)); feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(new Date(0)); feed.setDisabledUntil(Instant.EPOCH);
feedDAO.saveOrUpdate(feed); feedDAO.saveOrUpdate(feed);
} }
return feed; return feed;
@@ -54,7 +55,8 @@ public class FeedService {
String normalized = FeedUtils.normalizeURL(feed.getUrl()); String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized); feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized)); feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date()); feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.saveOrUpdate(feed); feedDAO.saveOrUpdate(feed);
} }

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.util.Date; import java.time.Instant;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -110,8 +110,8 @@ public class FeedSubscriptionService {
public void refreshAllUpForRefresh(User user) { public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user); List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) { for (FeedSubscription sub : subs) {
Date disabledUntil = sub.getFeed().getDisabledUntil(); Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.before(new Date())) { if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed(); Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed); feedRefreshEngine.refreshImmediately(feed);
} }

View File

@@ -1,9 +1,9 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Date;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
@@ -130,7 +130,7 @@ public class UserService {
byte[] salt = encryptionService.generateSalt(); byte[] salt = encryptionService.generateSalt();
user.setName(name); user.setName(name);
user.setEmail(email); user.setEmail(email);
user.setCreated(new Date()); user.setCreated(Instant.now());
user.setSalt(salt); user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt)); user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user); userDAO.saveOrUpdate(user);

View File

@@ -1,8 +1,7 @@
package com.commafeed.backend.service.internal; package com.commafeed.backend.service.internal;
import java.util.Date; import java.time.Instant;
import java.time.temporal.ChronoUnit;
import org.apache.commons.lang3.time.DateUtils;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
@@ -25,9 +24,9 @@ public class PostLoginActivities {
public void executeFor(User user) { public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in // only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Date now = new Date(); Instant now = Instant.now();
Date lastLogin = user.getLastLogin(); Instant lastLogin = user.getLastLogin();
if (lastLogin == null || lastLogin.before(DateUtils.addMinutes(now, -30))) { if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now); user.setLastLogin(now);
boolean heavyLoad = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad()); boolean heavyLoad = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad());

View File

@@ -0,0 +1,42 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

View File

@@ -1,5 +1,7 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
@@ -18,9 +20,10 @@ public class OldEntriesCleanupTask extends ScheduledTask {
@Override @Override
public void run() { public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); int maxAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxFeedCapacity > 0) { if (maxAgeDays > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity); Instant threshold = Instant.now().minus(Duration.ofDays(maxAgeDays));
cleaner.cleanEntriesOlderThan(threshold);
} }
} }

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.Date; import java.time.Instant;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
@@ -19,7 +19,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
@Override @Override
public void run() { public void run() {
Date threshold = config.getApplicationSettings().getUnreadThreshold(); Instant threshold = config.getApplicationSettings().getUnreadThreshold();
if (threshold != null) { if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold); cleaner.cleanStatusesOlderThan(threshold);
} }
@@ -27,7 +27,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 10; return 15;
} }
@Override @Override

View File

@@ -21,7 +21,7 @@ public class OrphanedContentsCleanupTask extends ScheduledTask {
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 20; return 25;
} }
@Override @Override

View File

@@ -21,7 +21,7 @@ public class OrphanedFeedsCleanupTask extends ScheduledTask {
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 15; return 20;
} }
@Override @Override

View File

@@ -2,12 +2,16 @@ package com.commafeed.frontend.auth;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Base64; import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.function.Function; import java.util.function.Function;
import org.glassfish.jersey.server.ContainerRequest; import org.glassfish.jersey.server.ContainerRequest;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole.Role; import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.UserService; import com.commafeed.backend.service.UserService;
@@ -25,7 +29,9 @@ public class SecurityCheckFactory implements Function<ContainerRequest, User> {
private static final String PREFIX = "Basic"; private static final String PREFIX = "Basic";
private final UserDAO userDAO;
private final UserService userService; private final UserService userService;
private final CommaFeedConfiguration config;
private final HttpServletRequest request; private final HttpServletRequest request;
private final Role role; private final Role role;
private final boolean apiKeyAllowed; private final boolean apiKeyAllowed;
@@ -45,21 +51,15 @@ public class SecurityCheckFactory implements Function<ContainerRequest, User> {
if (roles.contains(role)) { if (roles.contains(role)) {
return user.get(); return user.get();
} else { } else {
throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN) throw buildWebApplicationException(Response.Status.FORBIDDEN, "You don't have the required role to access this resource.");
.entity("You don't have the required role to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
} }
} else { } else {
throw new WebApplicationException(Response.status(Response.Status.UNAUTHORIZED) throw buildWebApplicationException(Response.Status.UNAUTHORIZED, "Credentials are required to access this resource.");
.entity("Credentials are required to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
} }
} }
Optional<User> cookieSessionLogin(SessionHelper sessionHelper) { Optional<User> cookieSessionLogin(SessionHelper sessionHelper) {
Optional<User> loggedInUser = sessionHelper.getLoggedInUser(); Optional<User> loggedInUser = sessionHelper.getLoggedInUserId().map(userDAO::findById);
loggedInUser.ifPresent(userService::performPostLoginActivities); loggedInUser.ifPresent(userService::performPostLoginActivities);
return loggedInUser; return loggedInUser;
} }
@@ -93,4 +93,11 @@ public class SecurityCheckFactory implements Function<ContainerRequest, User> {
return Optional.empty(); return Optional.empty();
} }
private WebApplicationException buildWebApplicationException(Response.Status status, String message) {
Map<String, Object> response = new HashMap<>();
response.put("message", message);
response.put("allowRegistrations", config.getApplicationSettings().getAllowRegistrations());
return new WebApplicationException(Response.status(status).entity(response).type(MediaType.APPLICATION_JSON).build());
}
} }

View File

@@ -9,6 +9,8 @@ import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractor
import org.glassfish.jersey.server.model.Parameter; import org.glassfish.jersey.server.model.Parameter;
import org.glassfish.jersey.server.spi.internal.ValueParamProvider; import org.glassfish.jersey.server.spi.internal.ValueParamProvider;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService; import com.commafeed.backend.service.UserService;
@@ -21,13 +23,17 @@ import lombok.RequiredArgsConstructor;
public class SecurityCheckFactoryProvider extends AbstractValueParamProvider { public class SecurityCheckFactoryProvider extends AbstractValueParamProvider {
private final UserService userService; private final UserService userService;
private final UserDAO userDAO;
private final CommaFeedConfiguration config;
private final HttpServletRequest request; private final HttpServletRequest request;
@Inject @Inject
public SecurityCheckFactoryProvider(final MultivaluedParameterExtractorProvider extractorProvider, UserService userService, public SecurityCheckFactoryProvider(final MultivaluedParameterExtractorProvider extractorProvider, UserDAO userDAO,
HttpServletRequest request) { UserService userService, CommaFeedConfiguration config, HttpServletRequest request) {
super(() -> extractorProvider, Parameter.Source.UNKNOWN); super(() -> extractorProvider, Parameter.Source.UNKNOWN);
this.userDAO = userDAO;
this.userService = userService; this.userService = userService;
this.config = config;
this.request = request; this.request = request;
} }
@@ -44,18 +50,22 @@ public class SecurityCheckFactoryProvider extends AbstractValueParamProvider {
return null; return null;
} }
return new SecurityCheckFactory(userService, request, securityCheck.value(), securityCheck.apiKeyAllowed()); return new SecurityCheckFactory(userDAO, userService, config, request, securityCheck.value(), securityCheck.apiKeyAllowed());
} }
@RequiredArgsConstructor @RequiredArgsConstructor
public static class Binder extends AbstractBinder { public static class Binder extends AbstractBinder {
private final UserDAO userDAO;
private final UserService userService; private final UserService userService;
private final CommaFeedConfiguration config;
@Override @Override
protected void configure() { protected void configure() {
bind(SecurityCheckFactoryProvider.class).to(ValueParamProvider.class).in(Singleton.class); bind(SecurityCheckFactoryProvider.class).to(ValueParamProvider.class).in(Singleton.class);
bind(userDAO).to(UserDAO.class);
bind(userService).to(UserService.class); bind(userService).to(UserService.class);
bind(config).to(CommaFeedConfiguration.class);
} }
} }

View File

@@ -1,6 +1,7 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable; import java.io.Serializable;
import java.time.Instant;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@@ -67,10 +68,10 @@ public class Entry implements Serializable {
private Integer mediaThumbnailHeight; private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED) @Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED)
private Date date; private Instant date;
@Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED) @Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED)
private Date insertedDate; private Instant insertedDate;
@Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED) @Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED)
private String feedId; private String feedId;
@@ -165,7 +166,7 @@ public class Entry implements Serializable {
} }
entry.setLink(getUrl()); entry.setLink(getUrl());
entry.setPublishedDate(getDate()); entry.setPublishedDate(getDate() == null ? null : Date.from(getDate()));
return entry; return entry;
} }
} }

View File

@@ -1,7 +1,7 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.Instant;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
@@ -30,10 +30,10 @@ public class Subscription implements Serializable {
private int errorCount; private int errorCount;
@Schema(description = "last time the feed was refreshed", type = "number") @Schema(description = "last time the feed was refreshed", type = "number")
private Date lastRefresh; private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number") @Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number")
private Date nextRefresh; private Instant nextRefresh;
@Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED) @Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl; private String feedUrl;
@@ -54,13 +54,12 @@ public class Subscription implements Serializable {
private int position; private int position;
@Schema(description = "date of the newest item", type = "number") @Schema(description = "date of the newest item", type = "number")
private Date newestItemTime; private Instant newestItemTime;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match") @Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter; private String filter;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) { public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
Date now = new Date();
FeedCategory category = subscription.getCategory(); FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed(); Feed feed = subscription.getFeed();
Subscription sub = new Subscription(); Subscription sub = new Subscription();
@@ -73,7 +72,8 @@ public class Subscription implements Serializable {
sub.setFeedLink(feed.getLink()); sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription)); sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated()); sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh((feed.getDisabledUntil() != null && feed.getDisabledUntil().before(now)) ? null : feed.getDisabledUntil()); sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount()); sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime()); sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId())); sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));

View File

@@ -1,7 +1,7 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.Instant;
import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data; import lombok.Data;
@@ -18,12 +18,12 @@ public class UnreadCount implements Serializable {
private long unreadCount; private long unreadCount;
@Schema(type = "number") @Schema(type = "number")
private Date newestItemTime; private Instant newestItemTime;
public UnreadCount() { public UnreadCount() {
} }
public UnreadCount(long feedId, long unreadCount, Date newestItemTime) { public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId; this.feedId = feedId;
this.unreadCount = unreadCount; this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime; this.newestItemTime = newestItemTime;

View File

@@ -1,7 +1,7 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.Instant;
import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode; import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
@@ -31,10 +31,10 @@ public class UserModel implements Serializable {
private boolean enabled; private boolean enabled;
@Schema(description = "account creation date", type = "number") @Schema(description = "account creation date", type = "number")
private Date created; private Instant created;
@Schema(description = "last login date", type = "number") @Schema(description = "last login date", type = "number")
private Date lastLogin; private Instant lastLogin;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED) @Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin; private boolean admin;

View File

@@ -1,9 +1,9 @@
package com.commafeed.frontend.resource; package com.commafeed.frontend.resource;
import java.io.StringWriter; import java.io.StringWriter;
import java.time.Instant;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
@@ -131,7 +131,7 @@ public class CategoryREST {
id = ALL; id = ALL;
} }
Date newerThanDate = newerThan == null ? null : new Date(newerThan); Instant newerThanDate = newerThan == null ? null : Instant.ofEpochMilli(newerThan);
List<Long> excludedIds = null; List<Long> excludedIds = null;
if (StringUtils.isNotEmpty(excludedSubscriptionIds)) { if (StringUtils.isNotEmpty(excludedSubscriptionIds)) {
@@ -242,8 +242,8 @@ public class CategoryREST {
Preconditions.checkNotNull(req); Preconditions.checkNotNull(req);
Preconditions.checkNotNull(req.getId()); Preconditions.checkNotNull(req.getId());
Date olderThan = req.getOlderThan() == null ? null : new Date(req.getOlderThan()); Instant olderThan = req.getOlderThan() == null ? null : Instant.ofEpochMilli(req.getOlderThan());
Date insertedBefore = req.getInsertedBefore() == null ? null : new Date(req.getInsertedBefore()); Instant insertedBefore = req.getInsertedBefore() == null ? null : Instant.ofEpochMilli(req.getInsertedBefore());
String keywords = req.getKeywords(); String keywords = req.getKeywords();
List<FeedEntryKeyword> entryKeywords = FeedEntryKeyword.fromQueryString(keywords); List<FeedEntryKeyword> entryKeywords = FeedEntryKeyword.fromQueryString(keywords);

View File

@@ -4,6 +4,7 @@ import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
@@ -164,7 +165,7 @@ public class FeedREST {
boolean unreadOnly = readType == ReadingMode.unread; boolean unreadOnly = readType == ReadingMode.unread;
Date newerThanDate = newerThan == null ? null : new Date(newerThan); Instant newerThanDate = newerThan == null ? null : Instant.ofEpochMilli(newerThan);
FeedSubscription subscription = feedSubscriptionDAO.findById(user, Long.valueOf(id)); FeedSubscription subscription = feedSubscriptionDAO.findById(user, Long.valueOf(id));
if (subscription != null) { if (subscription != null) {
@@ -245,8 +246,8 @@ public class FeedREST {
try { try {
FeedFetcherResult feedFetcherResult = feedFetcher.fetch(url, true, null, null, null, null); FeedFetcherResult feedFetcherResult = feedFetcher.fetch(url, true, null, null, null, null);
info = new FeedInfo(); info = new FeedInfo();
info.setUrl(feedFetcherResult.getUrlAfterRedirect()); info.setUrl(feedFetcherResult.urlAfterRedirect());
info.setTitle(feedFetcherResult.getTitle()); info.setTitle(feedFetcherResult.feed().title());
} catch (Exception e) { } catch (Exception e) {
log.debug(e.getMessage(), e); log.debug(e.getMessage(), e);
@@ -321,8 +322,8 @@ public class FeedREST {
Preconditions.checkNotNull(req); Preconditions.checkNotNull(req);
Preconditions.checkNotNull(req.getId()); Preconditions.checkNotNull(req.getId());
Date olderThan = req.getOlderThan() == null ? null : new Date(req.getOlderThan()); Instant olderThan = req.getOlderThan() == null ? null : Instant.ofEpochMilli(req.getOlderThan());
Date insertedBefore = req.getInsertedBefore() == null ? null : new Date(req.getInsertedBefore()); Instant insertedBefore = req.getInsertedBefore() == null ? null : Instant.ofEpochMilli(req.getInsertedBefore());
String keywords = req.getKeywords(); String keywords = req.getKeywords();
List<FeedEntryKeyword> entryKeywords = FeedEntryKeyword.fromQueryString(keywords); List<FeedEntryKeyword> entryKeywords = FeedEntryKeyword.fromQueryString(keywords);
@@ -379,7 +380,7 @@ public class FeedREST {
Calendar calendar = Calendar.getInstance(); Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.MONTH, 1); calendar.add(Calendar.MONTH, 1);
builder.expires(calendar.getTime()); builder.expires(calendar.getTime());
builder.lastModified(CommaFeedApplication.STARTUP_TIME); builder.lastModified(Date.from(CommaFeedApplication.STARTUP_TIME));
return builder.build(); return builder.build();
} }

View File

@@ -1,14 +1,14 @@
package com.commafeed.frontend.resource; package com.commafeed.frontend.resource;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections; import java.util.Collections;
import java.util.Date;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.hc.core5.net.URIBuilder; import org.apache.hc.core5.net.URIBuilder;
import com.codahale.metrics.annotation.Timed; import com.codahale.metrics.annotation.Timed;
@@ -153,7 +153,7 @@ public class UserREST {
s.setShowRead(settings.isShowRead()); s.setShowRead(settings.isShowRead());
s.setScrollMarks(settings.isScrollMarks()); s.setScrollMarks(settings.isScrollMarks());
s.setCustomCss(settings.getCustomCss()); s.setCustomCss(settings.getCustomCss());
s.setCustomJs(settings.getCustomJs()); s.setCustomJs(CommaFeedApplication.USERNAME_DEMO.equals(user.getName()) ? "" : settings.getCustomJs());
s.setLanguage(settings.getLanguage()); s.setLanguage(settings.getLanguage());
s.setScrollSpeed(settings.getScrollSpeed()); s.setScrollSpeed(settings.getScrollSpeed());
s.setAlwaysScrollToEntry(settings.isAlwaysScrollToEntry()); s.setAlwaysScrollToEntry(settings.isAlwaysScrollToEntry());
@@ -207,7 +207,7 @@ public class UserREST {
return Response.status(Status.FORBIDDEN).build(); return Response.status(Status.FORBIDDEN).build();
} }
Optional<User> login = userService.login(user.getEmail(), request.getCurrentPassword()); Optional<User> login = userService.login(user.getName(), request.getCurrentPassword());
if (login.isEmpty()) { if (login.isEmpty()) {
throw new BadRequestException("invalid password"); throw new BadRequestException("invalid password");
} }
@@ -282,7 +282,7 @@ public class UserREST {
try { try {
user.setRecoverPasswordToken(DigestUtils.sha1Hex(UUID.randomUUID().toString())); user.setRecoverPasswordToken(DigestUtils.sha1Hex(UUID.randomUUID().toString()));
user.setRecoverPasswordTokenDate(new Date()); user.setRecoverPasswordTokenDate(Instant.now());
userDAO.saveOrUpdate(user); userDAO.saveOrUpdate(user);
mailService.sendMail(user, "Password recovery", buildEmailContent(user)); mailService.sendMail(user, "Password recovery", buildEmailContent(user));
return Response.ok().build(); return Response.ok().build();
@@ -325,7 +325,7 @@ public class UserREST {
if (user.getRecoverPasswordToken() == null || !user.getRecoverPasswordToken().equals(token)) { if (user.getRecoverPasswordToken() == null || !user.getRecoverPasswordToken().equals(token)) {
return Response.status(Status.UNAUTHORIZED).entity("Invalid token.").build(); return Response.status(Status.UNAUTHORIZED).entity("Invalid token.").build();
} }
if (user.getRecoverPasswordTokenDate().before(DateUtils.addDays(new Date(), -2))) { if (ChronoUnit.DAYS.between(user.getRecoverPasswordTokenDate(), Instant.now()) >= 2) {
return Response.status(Status.UNAUTHORIZED).entity("token expired.").build(); return Response.status(Status.UNAUTHORIZED).entity("token expired.").build();
} }

View File

@@ -5,7 +5,6 @@ import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -189,7 +188,7 @@ public class FeverREST {
if (params.containsKey("mark") && params.containsKey("id") && params.containsKey("as")) { if (params.containsKey("mark") && params.containsKey("id") && params.containsKey("as")) {
long id = Long.parseLong(params.get("id")); long id = Long.parseLong(params.get("id"));
String before = params.get("before"); String before = params.get("before");
Date insertedBefore = before == null ? null : Date.from(Instant.ofEpochSecond(Long.parseLong(before))); Instant insertedBefore = before == null ? null : Instant.ofEpochSecond(Long.parseLong(before));
mark(user, params.get("mark"), id, params.get("as"), insertedBefore); mark(user, params.get("mark"), id, params.get("as"), insertedBefore);
} }
@@ -206,7 +205,7 @@ public class FeverREST {
.map(Feed::getLastUpdated) .map(Feed::getLastUpdated)
.filter(Objects::nonNull) .filter(Objects::nonNull)
.max(Comparator.naturalOrder()) .max(Comparator.naturalOrder())
.map(d -> d.toInstant().getEpochSecond()) .map(d -> d.getEpochSecond())
.orElse(0L); .orElse(0L);
} }
@@ -242,7 +241,7 @@ public class FeverREST {
f.setUrl(s.getFeed().getUrl()); f.setUrl(s.getFeed().getUrl());
f.setSiteUrl(s.getFeed().getLink()); f.setSiteUrl(s.getFeed().getLink());
f.setSpark(false); f.setSpark(false);
f.setLastUpdatedOnTime(s.getFeed().getLastUpdated() == null ? 0 : s.getFeed().getLastUpdated().toInstant().getEpochSecond()); f.setLastUpdatedOnTime(s.getFeed().getLastUpdated() == null ? 0 : s.getFeed().getLastUpdated().getEpochSecond());
return f; return f;
}).toList(); }).toList();
} }
@@ -291,7 +290,7 @@ public class FeverREST {
i.setUrl(s.getEntry().getUrl()); i.setUrl(s.getEntry().getUrl());
i.setSaved(s.isStarred()); i.setSaved(s.isStarred());
i.setRead(s.isRead()); i.setRead(s.isRead());
i.setCreatedOnTime(s.getEntryUpdated().toInstant().getEpochSecond()); i.setCreatedOnTime(s.getEntryUpdated().getEpochSecond());
return i; return i;
} }
@@ -306,7 +305,7 @@ public class FeverREST {
}).toList(); }).toList();
} }
private void mark(User user, String source, long id, String action, Date insertedBefore) { private void mark(User user, String source, long id, String action, Instant insertedBefore) {
if ("item".equals(source)) { if ("item".equals(source)) {
if ("read".equals(action) || "unread".equals(action)) { if ("read".equals(action) || "unread".equals(action)) {
feedEntryService.markEntry(user, id, "read".equals(action)); feedEntryService.markEntry(user, id, "read".equals(action));

View File

@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.Optional; import java.util.Optional;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserSettingsDAO; import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings; import com.commafeed.backend.model.UserSettings;
@@ -20,13 +21,16 @@ abstract class AbstractCustomCodeServlet extends HttpServlet {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserSettingsDAO userSettingsDAO; private final UserSettingsDAO userSettingsDAO;
@Override @Override
protected final void doGet(final HttpServletRequest req, HttpServletResponse resp) throws IOException { protected final void doGet(final HttpServletRequest req, HttpServletResponse resp) throws IOException {
resp.setContentType(getMimeType()); resp.setContentType(getMimeType());
final Optional<User> user = new SessionHelper(req).getLoggedInUser(); SessionHelper sessionHelper = new SessionHelper(req);
Optional<Long> userId = sessionHelper.getLoggedInUserId();
final Optional<User> user = unitOfWork.call(() -> userId.map(userDAO::findById));
if (user.isEmpty()) { if (user.isEmpty()) {
return; return;
} }

Some files were not shown because too many files have changed in this diff Show More