Compare commits

...

41 Commits
4.0.0 ... 4.1.0

Author SHA1 Message Date
Athou
874e7dcee6 release 4.1.0 2024-01-12 08:15:40 +01:00
Athou
8297edaf71 redirect to login page instead of welcome page if allowRegistrations is false (#1185) 2024-01-11 16:44:40 +01:00
Athou
9e4e629a1a prevent caching openapi files, so that the documentation is always up to date 2024-01-11 08:01:51 +01:00
Athou
8b86617f18 marking an entry as read/unread now requires to swipe to the left since swiping to the right now opens the mobile menu 2024-01-10 19:57:56 +01:00
Athou
bbda35f868 open sidebar on swipe (#1098) 2024-01-10 19:57:38 +01:00
Athou
df68405fef allow users without email to change their profile (#1184) 2024-01-10 19:28:03 +01:00
Athou
65194d948f ignore vscode files 2024-01-10 11:21:37 +01:00
Athou
d49297216c cleanup 2024-01-10 11:19:47 +01:00
Athou
e3e50f8456 improve artifact upload speed (https://github.com/actions/upload-artifact/issues/199) 2024-01-09 22:08:04 +01:00
Athou
e90b3730ef add JavaTimeModule to RedisCacheService object mapper to be able to serialize java.time.Instant 2024-01-09 22:01:34 +01:00
Athou
7675a24eb6 store only user id in session in order to avoid invalidating all sessions when user model changes 2024-01-09 21:22:20 +01:00
Athou
2bf9186135 only show sidebar resizer when sidebar is actually shown 2024-01-09 16:20:46 +01:00
Athou
d4ea51c145 fix vulnerability 2024-01-09 14:59:33 +01:00
Athou
6e0e99694e use properties file of git-commit-id-maven-plugin so we don't need to filter resources 2024-01-09 14:56:59 +01:00
Athou
9ede8d1c46 remove the Managed interface for classes that are not managed by dropwizard 2024-01-09 14:09:24 +01:00
Athou
fd0425a2be clear all sessions because the session model changed 2024-01-09 11:26:54 +01:00
Athou
2b976cadeb add a memory management section to the readme 2024-01-09 09:39:56 +01:00
Athou
023c27a565 setupListeners is only used for rtk query and we don't use it anymore 2024-01-09 07:24:24 +01:00
Athou
69c9988404 migrate from java.util.Date to java.time 2024-01-08 21:58:40 +01:00
Athou
b1a4debb95 replace toSorted usage with sort (#1183) 2024-01-08 13:48:27 +01:00
Athou
5663d619aa show category hierarchy (#1045) 2024-01-08 13:26:20 +01:00
Athou
2ef9e8d274 add null check 2024-01-07 22:14:00 +01:00
Athou
1292018de0 add setting to delete old entries 2024-01-07 20:49:02 +01:00
Athou
039e91414e prevent demo account from registering custom js code 2024-01-07 17:51:22 +01:00
Athou
662d0f754f avoid flash of light theme when using system color scheme 2024-01-07 17:51:22 +01:00
Athou
7fb7efbdf7 add missing truncate lost in refactoring 2024-01-07 17:51:22 +01:00
Athou
a841c80261 simplify trie building 2024-01-07 17:51:22 +01:00
Athou
da4143fa13 multiple feeds may have the same url hash 2024-01-07 17:51:22 +01:00
Athou
789857b09f compare feed entry content after cleanup because that's what saved in the database 2024-01-07 17:51:22 +01:00
Athou
ed45746f52 extract html cleaning code to its own service 2024-01-07 17:51:22 +01:00
Athou
deb51f2ccc rename FixedSizeSortedSet to FixedSizeSortedList because it's actually a list 2024-01-07 17:51:22 +01:00
Athou
5fec4a4c5f improve lookup by using a set because we only use contains() 2024-01-07 17:51:22 +01:00
Athou
7b335e2fd4 feed refresh engine now uses its own immutable model 2024-01-07 17:51:22 +01:00
Athou
60b6c69020 close the HTTP client after each test to close idle connections (https://github.com/dropwizard/dropwizard/issues/8174) 2024-01-06 08:37:12 +01:00
Athou
08ab32c4c2 we don't need the admin connector for tests 2024-01-05 21:20:56 +01:00
Athou
ff24fe4c7c eslint is already run by vite-plugin-eslint during build 2024-01-05 20:51:41 +01:00
Athou
50c62fb468 remove warning: 'typeParameters' property is deprecated 2024-01-05 20:48:25 +01:00
Athou
201331afc3 update vite to 5.x 2024-01-05 20:38:01 +01:00
Athou
cf3100081e add test for unauthorized websocket usage 2024-01-03 21:08:25 +01:00
Athou
860aab7495 fix typo 2024-01-02 11:11:45 +01:00
Athou
b084c8d108 remove line break 2024-01-02 11:10:33 +01:00
123 changed files with 2129 additions and 1608 deletions

View File

@@ -34,7 +34,7 @@ jobs:
run: mvn --batch-mode --update-snapshots verify
- name: Upload JAR
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ matrix.java == '17' }}
with:
name: commafeed.jar

3
.gitignore vendored
View File

@@ -35,5 +35,8 @@ src/main/app/lib
# Sublime
*.sublime*
# VSCode
.vscode
# Macs
*.DS_Store

View File

@@ -1,5 +1,24 @@
# Changelog
## [4.1.0]
- it is now possible to open the sidebar on mobile by swiping to the right (#1098)
- swiping to mark entries as read/unread changed from swipinig right to left because swiping right now opens the sidebar
- the full hierarchy of categories are now displayed in the category dropdown (#1045)
- added a setting `maxEntriesAgeDays` to delete old entries based on their age during database cleanup.
The setting is disabled by default for existing installations, except for the docker image where it is enabled and set
to 365 days
- if user registrations are disabled on your instance which is the default behavior, users are redirected on the login
page instead of the welcome page when not logged in (#1185)
- the sidebar resizer is no longer shown in the middle of the screen on mobile
- when using the system color scheme and the system is using a dark theme, feed entries no longer flicker on load
- the demo account (if enabled) cannot register custom javascript code anymore
- removed the usage of `toSorted` in the client because older browsers do not support it (#1183)
- the openapi documentation is no longer cached by the browser so you always have access to the latest version
- added a memory management section to the readme, reading it is recommended if you are running CommaFeed on a server
with limited memory
- fixed an issue that caused users without an email address set to be unable to edit their profile (#1184)
## [4.0.0]
- migrated from dropwizard 2 to dropwizard 4, Java 17+ is now required
@@ -11,14 +30,13 @@
- custom JS code is now executed when the app is done loading instead of when the page is loaded
- the favicon is now correctly returned for feeds that return an invalid content type
- the feed refresh engine now uses httpclient5 with connection pooling and no longer creates a new client for each
request,
reducing CPU usage
request, reducing CPU usage
- updated UI library Mantine to 7.0, improving performance
- the h2 embedded database is now compacted on shutdown to reclaim unused space
- the admin connector on port 8084 is now disabled in config.yml.example. Disabling it in your config.yml is
recommended (see https://github.com/Athou/commafeed/commit/929df60f09cce56020b0962ab111cd8349b271b0)
- migrated documentation from swagger 2 to openapi 3
- added a GET method to the fever api to indicate that the endpoint is working correctly when accesed from a browser
- added a GET method to the fever api to indicate that the endpoint is working correctly when accessed from a browser
- the websocket connection can now be disabled, the websocket ping interval and the tree reload interval can now be
configured (see config.yml.example)
- the websocket connection now works correctly when the context root of the application is not "/"

View File

@@ -54,6 +54,29 @@ user is `admin` and the default password is `admin`.
The server will listen on http://localhost:8082. The default
user is `admin` and the default password is `admin`.
### Memory management
The Java Virtual Machine (JVM) is rather greedy by default and will not release unused memory to the
operating system. This is because acquiring memory from the operating system is a relatively expensive operation.
However, this can be problematic on systems with limited memory.
#### Hard limit
The JVM can be configured to use a maximum amount of memory with the `-Xmx` parameter.
For example, to limit the JVM to 256MB of memory, use `-Xmx256m`.
#### Dynamic sizing
The JVM can be configured to release unused memory to the operating system with the following parameters:
-Xms20m -XX:+UseG1GC -XX:-ShrinkHeapInSteps -XX:G1PeriodicGCInterval=10000 -XX:-G1PeriodicGCInvokesConcurrent -XX:MinHeapFreeRatio=5 -XX:MaxHeapFreeRatio=10
This is how the Docker image is configured.
See [here](https://docs.oracle.com/en/java/javase/17/gctuning/garbage-first-g1-garbage-collector1.html)
and [here](https://docs.oracle.com/en/java/javase/17/gctuning/factors-affecting-garbage-collection-performance.html) for
more
information.
## Translation
Files for internationalization are

File diff suppressed because it is too large Load Diff

View File

@@ -16,9 +16,9 @@
"dependencies": {
"@emotion/react": "^11.11.3",
"@fontsource/open-sans": "^5.0.20",
"@lingui/core": "^4.6.0",
"@lingui/macro": "^4.6.0",
"@lingui/react": "^4.6.0",
"@lingui/core": "^4.7.0",
"@lingui/macro": "^4.7.0",
"@lingui/react": "^4.7.0",
"@mantine/core": "^7.3.2",
"@mantine/form": "^7.3.2",
"@mantine/hooks": "^7.3.2",
@@ -52,8 +52,8 @@
"websocket-heartbeat-js": "^1.1.3"
},
"devDependencies": {
"@lingui/cli": "^4.6.0",
"@lingui/vite-plugin": "^4.6.0",
"@lingui/cli": "^4.7.0",
"@lingui/vite-plugin": "^4.7.0",
"@types/mousetrap": "^1.6.15",
"@types/react": "^18.2.46",
"@types/react-dom": "^18.2.18",
@@ -76,10 +76,10 @@
"prettier": "^3.1.1",
"rollup-plugin-visualizer": "^5.12.0",
"typescript": "^5.3.3",
"vite": "^4.5.1",
"vite": "^5.0.11",
"vite-plugin-eslint": "^1.8.1",
"vite-tsconfig-paths": "^4.2.3",
"vitest": "^0.34.6",
"vitest": "^1.1.3",
"vitest-mock-extended": "^1.3.1"
}
}

View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId>
<version>4.0.0</version>
<version>4.1.0</version>
</parent>
<artifactId>commafeed-client</artifactId>
<name>CommaFeed Client</name>
@@ -39,16 +39,6 @@
<arguments>ci</arguments>
</configuration>
</execution>
<execution>
<id>npm run eslint</id>
<goals>
<goal>npm</goal>
</goals>
<phase>compile</phase>
<configuration>
<arguments>run eslint</arguments>
</configuration>
</execution>
<execution>
<id>npm run test</id>
<goals>

View File

@@ -31,11 +31,12 @@ const axiosInstance = axios.create({ baseURL: "./rest", withCredentials: true })
axiosInstance.interceptors.response.use(
response => response,
error => {
const { status, data } = error.response
if (
(error.response.status === 401 && error.response.data === "Credentials are required to access this resource.") ||
(error.response.status === 403 && error.response.data === "You don't have the required role to access this resource.")
(status === 401 && data?.message === "Credentials are required to access this resource.") ||
(status === 403 && data?.message === "You don't have the required role to access this resource.")
) {
window.location.hash = "/welcome"
window.location.hash = data?.allowRegistrations ? "/welcome" : "/login"
}
throw error
}

View File

@@ -1,5 +1,4 @@
import { configureStore } from "@reduxjs/toolkit"
import { setupListeners } from "@reduxjs/toolkit/query"
import { entriesSlice } from "app/entries/slice"
import { redirectSlice } from "app/redirect/slice"
import { serverSlice } from "app/server/slice"
@@ -17,8 +16,6 @@ export const reducers = {
export const store = configureStore({ reducer: reducers })
setupListeners(store.dispatch)
export type RootState = ReturnType<typeof store.getState>
export type AppDispatch = typeof store.dispatch

View File

@@ -110,7 +110,7 @@ export function KeyboardShortcutsHelp() {
<Table.Td>
<Kbd>M</Kbd>
<span>, </span>
<Trans>Swipe header to the right</Trans>
<Trans>Swipe header to the left</Trans>
</Table.Td>
</Table.Tr>
<Table.Tr>

View File

@@ -91,7 +91,7 @@ export function FeedEntries() {
)
}
const swipedRight = async (entry: ExpendableEntry) => await dispatch(markEntry({ entry, read: !entry.read }))
const swipedLeft = async (entry: ExpendableEntry) => await dispatch(markEntry({ entry, read: !entry.read }))
// close context menu on scroll
useEffect(() => {
@@ -320,7 +320,7 @@ export function FeedEntries() {
onHeaderClick={event => headerClicked(entry, event)}
onHeaderRightClick={event => headerRightClicked(entry, event)}
onBodyClick={() => bodyClicked(entry)}
onSwipedRight={async () => await swipedRight(entry)}
onSwipedLeft={async () => await swipedLeft(entry)}
/>
</div>
))}

View File

@@ -21,7 +21,7 @@ interface FeedEntryProps {
onHeaderClick: (e: React.MouseEvent) => void
onHeaderRightClick: (e: React.MouseEvent) => void
onBodyClick: (e: React.MouseEvent) => void
onSwipedRight: () => void
onSwipedLeft: () => void
}
const useStyles = tss
@@ -111,7 +111,7 @@ export function FeedEntry(props: FeedEntryProps) {
})
const swipeHandlers = useSwipeable({
onSwipedRight: props.onSwipedRight,
onSwipedLeft: props.onSwipedLeft,
})
let paddingX: MantineSpacing = "xs"

View File

@@ -3,6 +3,7 @@ import { Select, type SelectProps } from "@mantine/core"
import { type ComboboxItem } from "@mantine/core/lib/components/Combobox/Combobox.types"
import { Constants } from "app/constants"
import { useAppSelector } from "app/store"
import { type Category } from "app/types"
import { flattenCategoryTree } from "app/utils"
type CategorySelectProps = Partial<SelectProps> & {
@@ -13,14 +14,32 @@ type CategorySelectProps = Partial<SelectProps> & {
export function CategorySelect(props: CategorySelectProps) {
const rootCategory = useAppSelector(state => state.tree.rootCategory)
const categories = rootCategory && flattenCategoryTree(rootCategory)
const categoriesById = categories?.reduce((map, c) => {
map.set(c.id, c)
return map
}, new Map<string, Category>())
const categoryLabel = (cat: Category) => {
let label = cat.name
while (cat.parentId) {
const parent = categoriesById?.get(cat.parentId)
if (!parent) {
break
}
label = `${parent.name}${label}`
cat = parent
}
return label
}
const selectData: ComboboxItem[] | undefined = categories
?.filter(c => c.id !== Constants.categories.all.id)
.filter(c => !props.withoutCategoryIds?.includes(c.id))
.sort((c1, c2) => c1.name.localeCompare(c2.name))
.map(c => ({
label: c.parentName ? t`${c.name} (in ${c.parentName})` : c.name,
label: categoryLabel(c),
value: c.id,
}))
.sort((c1, c2) => c1.label.localeCompare(c2.label))
if (props.withAll) {
selectData?.unshift({
label: t`All`,

View File

@@ -16,13 +16,13 @@ export function TreeSearch(props: TreeSearchProps) {
const dispatch = useAppDispatch()
const actions: SpotlightActionData[] = props.feeds
.toSorted((f1, f2) => f1.name.localeCompare(f2.name))
.map(f => ({
id: `${f.id}`,
label: f.name,
leftSection: <FeedFavicon url={f.iconUrl} />,
onClick: async () => await dispatch(redirectToFeed(f.id)),
}))
.sort((f1, f2) => f1.label.localeCompare(f2.label))
const searchIcon = <TbSearch size={18} />
const rightSection = (

View File

@@ -1,4 +1,20 @@
import { useComputedColorScheme } from "@mantine/core"
// the color scheme to use to render components
export const useColorScheme = () => useComputedColorScheme("light")
import { useMantineColorScheme } from "@mantine/core"
import { useMediaQuery } from "@mantine/hooks"
export const useColorScheme = () => {
const systemColorScheme = useMediaQuery(
"(prefers-color-scheme: dark)",
// passing undefined will use window.matchMedia(query) as default value
undefined,
{
// get initial value synchronously and not in useEffect to avoid flash of light theme
getInitialValueInEffect: false,
}
)
? "dark"
: "light"
const { colorScheme } = useMantineColorScheme()
return colorScheme === "auto" ? systemColorScheme : colorScheme
}

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "النجاح"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Éxit"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Úspěch"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Llwyddiant"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Succes"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "CommaFeed ist ein Open Source Projekt. Der Quellcode wird auf auf </0><1>GitHub</1> gehostet."
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Erfolg"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr "{0} (in {1})"
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
@@ -823,8 +819,8 @@ msgid "Success"
msgstr "Success"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgstr "Swipe header to the right"
msgid "Swipe header to the left"
msgstr "Swipe header to the left"
#: src/pages/WelcomePage.tsx
msgid "Switch to dark theme"

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Éxito"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "موفقیت"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Onnistui"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr "{0} (sur {1})"
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "<0>CommaFeed est un projet open-source. Les sources sont hébergées sur </0><1>GitHub</1>."
@@ -823,8 +819,8 @@ msgid "Success"
msgstr "Succès"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgstr "Faire glisser le titre vers la droite"
msgid "Swipe header to the left"
msgstr "Faire glisser le titre vers la gauche"
#: src/pages/WelcomePage.tsx
msgid "Switch to dark theme"

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Éxito"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Siker"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Sukses"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Successo"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "成功"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "성공"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Kejayaan"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Suksess"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Succes"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Suksess"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Sukces"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Sucesso"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Успех"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Úspech"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "Framgång"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr "{0} ({1} içinde)"
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr "<0>CommaFeed açık kaynak kodlu bir proje. Kaynak kodları </0><1>GitHub</1>'da."
@@ -823,8 +819,8 @@ msgid "Success"
msgstr "Başarı"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgstr "Başlığı sağa kaydır"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx
msgid "Switch to dark theme"

View File

@@ -13,10 +13,6 @@ msgstr ""
"Language-Team: \n"
"Plural-Forms: \n"
#: src/components/content/add/CategorySelect.tsx
msgid "{0} (in {1})"
msgstr ""
#: src/pages/app/AboutPage.tsx
msgid "<0>CommaFeed is an open-source project. Sources are hosted on </0><1>GitHub</1>."
msgstr ""
@@ -823,7 +819,7 @@ msgid "Success"
msgstr "成功"
#: src/components/KeyboardShortcutsHelp.tsx
msgid "Swipe header to the right"
msgid "Swipe header to the left"
msgstr ""
#: src/pages/WelcomePage.tsx

View File

@@ -19,6 +19,7 @@ import { type ReactNode, Suspense, useEffect } from "react"
import Draggable from "react-draggable"
import { TbMenu2, TbPlus, TbX } from "react-icons/tb"
import { Outlet } from "react-router-dom"
import { useSwipeable } from "react-swipeable"
import { tss } from "tss"
import useLocalStorage from "use-local-storage"
@@ -111,81 +112,94 @@ export default function Layout(props: LayoutProps) {
</ActionIcon>
)
const swipeHandlers = useSwipeable({
onSwiping: e => {
const threshold = document.documentElement.clientWidth / 6
if (e.absX > threshold) {
dispatch(setMobileMenuOpen(e.dir === "Right"))
}
},
})
if (loading) return <LoadingPage />
return (
<AppShell
header={{ height: Constants.layout.headerHeight }}
navbar={{
width: sidebarWidth,
breakpoint: Constants.layout.mobileBreakpoint,
collapsed: { mobile: !mobileMenuOpen, desktop: !props.sidebarVisible },
}}
padding={{ base: 6, [Constants.layout.mobileBreakpointName]: "md" }}
>
<AppShell.Header id="header">
<OnMobile>
{mobileMenuOpen && (
<Group justify="space-between" p="md">
<Box>{burger}</Box>
<Box>
<LogoAndTitle />
</Box>
<Box>{addButton}</Box>
</Group>
)}
{!mobileMenuOpen && (
<Box {...swipeHandlers}>
<AppShell
header={{ height: Constants.layout.headerHeight }}
navbar={{
width: sidebarWidth,
breakpoint: Constants.layout.mobileBreakpoint,
collapsed: { mobile: !mobileMenuOpen, desktop: !props.sidebarVisible },
}}
padding={{ base: 6, [Constants.layout.mobileBreakpointName]: "md" }}
>
<AppShell.Header id="header">
<OnMobile>
{mobileMenuOpen && (
<Group justify="space-between" p="md">
<Box>{burger}</Box>
<Box>
<LogoAndTitle />
</Box>
<Box>{addButton}</Box>
</Group>
)}
{!mobileMenuOpen && (
<Group p="md">
<Box>{burger}</Box>
<Box style={{ flexGrow: 1 }}>{props.header}</Box>
</Group>
)}
</OnMobile>
<OnDesktop>
<Group p="md">
<Box>{burger}</Box>
<Group justify="space-between" style={{ width: sidebarWidth - 16 }}>
<Box>
<LogoAndTitle />
</Box>
<Box>{addButton}</Box>
</Group>
<Box style={{ flexGrow: 1 }}>{props.header}</Box>
</Group>
)}
</OnMobile>
</OnDesktop>
</AppShell.Header>
<AppShell.Navbar id="sidebar" p={sidebarPadding}>
<AppShell.Section grow component={ScrollArea} mx="-sm" px="sm">
<Box className={classes.sidebarContent}>{props.sidebar}</Box>
</AppShell.Section>
</AppShell.Navbar>
<OnDesktop>
<Group p="md">
<Group justify="space-between" style={{ width: sidebarWidth - 16 }}>
<Box>
<LogoAndTitle />
</Box>
<Box>{addButton}</Box>
</Group>
<Box style={{ flexGrow: 1 }}>{props.header}</Box>
</Group>
<Draggable
axis="x"
defaultPosition={{
x: sidebarWidth,
y: Constants.layout.headerHeight,
}}
bounds={{
left: 120,
right: 1000,
}}
grid={[30, 30]}
onDrag={(_e, data) => setSidebarWidth(data.x)}
>
<Box
style={{
position: "fixed",
height: "100%",
width: "10px",
cursor: "ew-resize",
}}
></Box>
</Draggable>
</OnDesktop>
</AppShell.Header>
<AppShell.Navbar id="sidebar" p={sidebarPadding}>
<AppShell.Section grow component={ScrollArea} mx="-sm" px="sm">
<Box className={classes.sidebarContent}>{props.sidebar}</Box>
</AppShell.Section>
</AppShell.Navbar>
<Draggable
axis="x"
defaultPosition={{
x: sidebarWidth,
y: Constants.layout.headerHeight,
}}
bounds={{
left: 120,
right: 1000,
}}
grid={[30, 30]}
onDrag={(_e, data) => setSidebarWidth(data.x)}
>
<Box
style={{
position: "fixed",
height: "100%",
width: "10px",
cursor: "ew-resize",
}}
></Box>
</Draggable>
<AppShell.Main id="content">
<Suspense fallback={<Loader />}>
<AnnouncementDialog />
<Outlet />
</Suspense>
</AppShell.Main>
</AppShell>
<AppShell.Main id="content">
<Suspense fallback={<Loader />}>
<AnnouncementDialog />
<Outlet />
</Suspense>
</AppShell.Main>
</AppShell>
</Box>
)
}

View File

@@ -6,7 +6,7 @@ import eslint from "vite-plugin-eslint"
import tsconfigPaths from "vite-tsconfig-paths"
// https://vitejs.dev/config/
export default defineConfig({
export default defineConfig(env => ({
plugins: [
react({
babel: {
@@ -15,7 +15,8 @@ export default defineConfig({
},
}),
lingui(),
eslint(),
// https://github.com/vitest-dev/vitest/issues/4055#issuecomment-1732994672
env.mode !== "test" && eslint(),
tsconfigPaths(),
visualizer(),
],
@@ -32,7 +33,7 @@ export default defineConfig({
},
},
build: {
chunkSizeWarningLimit: 3000,
chunkSizeWarningLimit: 3500,
rollupOptions: {
output: {
manualChunks: id => {
@@ -44,4 +45,4 @@ export default defineConfig({
},
},
},
})
}))

View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 365
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0

View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 365
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0

View File

@@ -6,7 +6,7 @@
<parent>
<groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId>
<version>4.0.0</version>
<version>4.1.0</version>
</parent>
<artifactId>commafeed-server</artifactId>
<name>CommaFeed Server</name>
@@ -31,12 +31,6 @@
<build>
<finalName>commafeed</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
@@ -69,7 +63,8 @@
</execution>
</executions>
<configuration>
<generateGitPropertiesFile>false</generateGitPropertiesFile>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
<failOnNoGitDirectory>false</failOnNoGitDirectory>
<failOnUnableToExtractRepoInfo>false</failOnUnableToExtractRepoInfo>
</configuration>
@@ -216,7 +211,7 @@
<dependency>
<groupId>com.commafeed</groupId>
<artifactId>commafeed-client</artifactId>
<version>4.0.0</version>
<version>4.1.0</version>
</dependency>
<dependency>

View File

@@ -1,7 +1,7 @@
package com.commafeed;
import java.io.IOException;
import java.util.Date;
import java.time.Instant;
import java.util.EnumSet;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
@@ -10,6 +10,7 @@ import java.util.concurrent.TimeUnit;
import org.hibernate.cfg.AvailableSettings;
import com.codahale.metrics.json.MetricsModule;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
@@ -42,8 +43,10 @@ import com.commafeed.frontend.servlet.RobotsTxtDisallowAllServlet;
import com.commafeed.frontend.session.SessionHelperFactoryProvider;
import com.commafeed.frontend.ws.WebSocketConfigurator;
import com.commafeed.frontend.ws.WebSocketEndpoint;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
@@ -76,7 +79,7 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
public static final Date STARTUP_TIME = new Date();
public static final Instant STARTUP_TIME = Instant.now();
private HibernateBundle<CommaFeedConfiguration> hibernateBundle;
private WebsocketBundle<CommaFeedConfiguration> websocketBundle;
@@ -89,8 +92,7 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
@Override
public void initialize(Bootstrap<CommaFeedConfiguration> bootstrap) {
configureEnvironmentSubstitutor(bootstrap);
bootstrap.getObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
configureObjectMapper(bootstrap.getObjectMapper());
bootstrap.addBundle(websocketBundle = new WebsocketBundle<>());
bootstrap.addBundle(hibernateBundle = new HibernateBundle<>(AbstractModel.class, Feed.class, FeedCategory.class, FeedEntry.class,
@@ -134,6 +136,15 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
bootstrap.setConfigurationSourceProvider(buildEnvironmentSubstitutor(bootstrap));
}
private static void configureObjectMapper(ObjectMapper objectMapper) {
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
private static EnvironmentSubstitutor buildEnvironmentSubstitutor(Bootstrap<CommaFeedConfiguration> bootstrap) {
// enable config.yml string substitution
// e.g. having a custom config.yml file with app.session.path=${SOME_ENV_VAR} will substitute SOME_ENV_VAR
@@ -156,7 +167,9 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
environment.servlets().setSessionHandler(config.getSessionHandlerFactory().build(config.getDataSourceFactory()));
// support for "@SecurityCheck User user" injection
environment.jersey().register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserService.class)));
environment.jersey()
.register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserDAO.class),
injector.getInstance(UserService.class), config));
// support for "@Context SessionHelper sessionHelper" injection
environment.jersey().register(new SessionHelperFactoryProvider.Binder());
@@ -208,6 +221,12 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
environment.servlets()
.addFilter("index-cache-busting-filter", new CacheBustingFilter())
.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/");
// prevent caching openapi files, so that the documentation is always up to date
environment.servlets()
.addFilter("openapi-cache-busting-filter", new CacheBustingFilter())
.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/openapi.json", "/openapi.yaml");
// prevent caching REST resources, except for favicons
environment.servlets().addFilter("rest-cache-busting-filter", new CacheBustingFilter() {
@Override

View File

@@ -1,9 +1,10 @@
package com.commafeed;
import java.util.Date;
import java.util.ResourceBundle;
import org.apache.commons.lang3.time.DateUtils;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Properties;
import com.commafeed.backend.cache.RedisPoolFactory;
import com.commafeed.frontend.session.SessionHandlerFactory;
@@ -54,10 +55,17 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
private final String gitCommit;
public CommaFeedConfiguration() {
ResourceBundle bundle = ResourceBundle.getBundle("application");
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
this.version = bundle.getString("version");
this.gitCommit = bundle.getString("git.commit");
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
@Override
@@ -146,6 +154,11 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
@Valid
private Integer maxFeedCapacity;
@NotNull
@Min(0)
@Valid
private Integer maxEntriesAgeDays = 0;
@NotNull
@Valid
private Integer maxFeedsPerUser = 0;
@@ -170,9 +183,8 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
private Duration treeReloadInterval = Duration.seconds(30);
public Date getUnreadThreshold() {
int keepStatusDays = getKeepStatusDays();
return keepStatusDays > 0 ? DateUtils.addDays(new Date(), -1 * keepStatusDays) : null;
public Instant getUnreadThreshold() {
return getKeepStatusDays() > 0 ? Instant.now().minus(getKeepStatusDays(), ChronoUnit.DAYS) : null;
}
}

View File

@@ -19,6 +19,7 @@ import com.commafeed.backend.favicon.DefaultFaviconFetcher;
import com.commafeed.backend.favicon.FacebookFaviconFetcher;
import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
import com.commafeed.backend.task.DemoAccountCleanupTask;
import com.commafeed.backend.task.EntriesExceedingFeedCapacityCleanupTask;
import com.commafeed.backend.task.OldEntriesCleanupTask;
import com.commafeed.backend.task.OldStatusesCleanupTask;
import com.commafeed.backend.task.OrphanedContentsCleanupTask;
@@ -66,6 +67,7 @@ public class CommaFeedModule extends AbstractModule {
Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
taskMultibinder.addBinding().to(EntriesExceedingFeedCapacityCleanupTask.class);
taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);

View File

@@ -10,14 +10,14 @@ import java.util.List;
*
*
*/
public class FixedSizeSortedSet<E> {
public class FixedSizeSortedList<E> {
private final List<E> inner;
private final Comparator<? super E> comparator;
private final int capacity;
public FixedSizeSortedSet(int capacity, Comparator<? super E> comparator) {
public FixedSizeSortedList(int capacity, Comparator<? super E> comparator) {
this.inner = new ArrayList<>(Math.max(0, capacity));
this.capacity = capacity < 0 ? Integer.MAX_VALUE : capacity;
this.comparator = comparator;

View File

@@ -31,7 +31,6 @@ import com.commafeed.CommaFeedConfiguration;
import com.google.common.collect.Iterables;
import com.google.common.net.HttpHeaders;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.Getter;
@@ -44,7 +43,7 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils;
*
*/
@Singleton
public class HttpGetter implements Managed {
public class HttpGetter {
private final CloseableHttpClient client;
@@ -154,11 +153,6 @@ public class HttpGetter implements Managed {
.build();
}
@Override
public void stop() throws Exception {
client.close();
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;

View File

@@ -1,11 +1,12 @@
package com.commafeed.backend.cache;
import java.util.List;
import java.util.Set;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
@@ -14,12 +15,12 @@ import com.commafeed.frontend.model.UnreadCount;
public abstract class CacheService {
// feed entries for faster refresh
public abstract List<String> getLastEntries(Feed feed);
public abstract Set<String> getLastEntries(Feed feed);
public abstract void setLastEntries(Feed feed, List<String> entries);
public String buildUniqueEntryKey(Feed feed, FeedEntry entry) {
return DigestUtils.sha1Hex(entry.getGuid() + entry.getUrl());
public String buildUniqueEntryKey(Entry entry) {
return DigestUtils.sha1Hex(entry.guid() + entry.url());
}
// user categories

View File

@@ -2,6 +2,7 @@ package com.commafeed.backend.cache;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
@@ -12,8 +13,8 @@ import com.commafeed.frontend.model.UnreadCount;
public class NoopCacheService extends CacheService {
@Override
public List<String> getLastEntries(Feed feed) {
return Collections.emptyList();
public Set<String> getLastEntries(Feed feed) {
return Collections.emptySet();
}
@Override

View File

@@ -1,6 +1,5 @@
package com.commafeed.backend.cache;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -13,6 +12,7 @@ import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -24,19 +24,16 @@ import redis.clients.jedis.Pipeline;
@RequiredArgsConstructor
public class RedisCacheService extends CacheService {
private static final ObjectMapper MAPPER = new ObjectMapper();
private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
private final JedisPool pool;
@Override
public List<String> getLastEntries(Feed feed) {
List<String> list = new ArrayList<>();
public Set<String> getLastEntries(Feed feed) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Set<String> members = jedis.smembers(key);
list.addAll(members);
return jedis.smembers(key);
}
return list;
}
@Override

View File

@@ -1,16 +1,14 @@
package com.commafeed.backend.dao;
import java.util.Date;
import java.time.Instant;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
@@ -28,8 +26,8 @@ public class FeedDAO extends GenericDAO<Feed> {
super(sessionFactory);
}
public List<Feed> findNextUpdatable(int count, Date lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(feed).where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(new Date())));
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(feed).where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.where(JPAExpressions.selectOne()
.from(subscription)
@@ -41,17 +39,18 @@ public class FeedDAO extends GenericDAO<Feed> {
return query.orderBy(feed.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Date date) {
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(feed).set(feed.disabledUntil, date).where(feed.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl) {
List<Feed> feeds = query().selectFrom(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).fetch();
Feed feed = Iterables.getFirst(feeds, null);
if (feed != null && StringUtils.equals(normalizedUrl, feed.getNormalizedUrl())) {
return feed;
}
return null;
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(feed)
.where(feed.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> StringUtils.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {

View File

@@ -1,8 +1,8 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
@@ -26,12 +26,8 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
super(sessionFactory);
}
public Long findExisting(String guid, Feed feed) {
return query().select(entry.id)
.from(entry)
.where(entry.guidHash.eq(DigestUtils.sha1Hex(guid)), entry.feed.eq(feed))
.limit(1)
.fetchOne();
public Long findExisting(String guidHash, Feed feed) {
return query().select(entry.id).from(entry).where(entry.guidHash.eq(guidHash), entry.feed.eq(feed)).limit(1).fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
@@ -50,6 +46,17 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.updated.lt(olderThan)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);

View File

@@ -1,8 +1,8 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
@@ -10,7 +10,7 @@ import org.apache.commons.lang3.builder.CompareToBuilder;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.FixedSizeSortedSet;
import com.commafeed.backend.FixedSizeSortedList;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
@@ -73,8 +73,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
boolean read = unreadThreshold != null && entry.getUpdated().before(unreadThreshold);
Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
boolean read = unreadThreshold != null && entry.getUpdated().isBefore(unreadThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
@@ -90,7 +90,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
return status;
}
public List<FeedEntryStatus> findStarred(User user, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent) {
public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue());
if (newerThan != null) {
query.where(status.entryInserted.gt(newerThan));
@@ -114,7 +115,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
}
private JPAQuery<FeedEntry> buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List<FeedEntryKeyword> keywords,
Date newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag, Long minEntryId, Long maxEntryId) {
Instant newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag, Long minEntryId,
Long maxEntryId) {
JPAQuery<FeedEntry> query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed()));
@@ -139,7 +141,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
or.or(status.read.isFalse());
query.where(or);
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
if (unreadThreshold != null) {
query.where(entry.updated.goe(unreadThreshold));
}
@@ -193,22 +195,22 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
boolean onlyIds, String tag, Long minEntryId, Long maxEntryId) {
int capacity = offset + limit;
Comparator<FeedEntryStatus> comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC;
FixedSizeSortedSet<FeedEntryStatus> set = new FixedSizeSortedSet<>(capacity, comparator);
FixedSizeSortedList<FeedEntryStatus> fssl = new FixedSizeSortedList<>(capacity, comparator);
for (FeedSubscription sub : subs) {
FeedEntryStatus last = (order != null && set.isFull()) ? set.last() : null;
FeedEntryStatus last = (order != null && fssl.isFull()) ? fssl.last() : null;
JPAQuery<FeedEntry> query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag, minEntryId,
maxEntryId);
List<Tuple> tuples = query.select(entry.id, entry.updated, status.id, entry.content.title).fetch();
for (Tuple tuple : tuples) {
Long id = tuple.get(entry.id);
Date updated = tuple.get(entry.updated);
Instant updated = tuple.get(entry.updated);
Long statusId = tuple.get(status.id);
FeedEntryContent content = new FeedEntryContent();
@@ -225,11 +227,11 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
status.setEntry(entry);
status.setSubscription(sub);
set.add(status);
fssl.add(status);
}
}
List<FeedEntryStatus> placeholders = set.asList();
List<FeedEntryStatus> placeholders = fssl.asList();
int size = placeholders.size();
if (size < offset) {
return new ArrayList<>();
@@ -260,7 +262,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
List<Tuple> tuples = query.select(entry.count(), entry.updated.max()).fetch();
for (Tuple tuple : tuples) {
Long count = tuple.get(entry.count());
Date updated = tuple.get(entry.updated.max());
Instant updated = tuple.get(entry.updated.max());
uc = new UnreadCount(subscription.getId(), count == null ? 0 : count, updated);
}
return uc;
@@ -276,7 +278,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
return results;
}
public long deleteOldStatuses(Date olderThan, int limit) {
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(status.id)
.from(status)
.where(status.entryInserted.lt(olderThan), status.starred.isFalse())

View File

@@ -1,8 +1,7 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.time.Instant;
import java.util.Set;
import org.apache.commons.codec.binary.StringUtils;
@@ -11,16 +10,14 @@ import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedParser.FeedParserResult;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
/**
@@ -35,8 +32,8 @@ public class FeedFetcher {
private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders;
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate,
String lastContentHash) throws FeedException, IOException, NotModifiedException {
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
int timeout = 20000;
@@ -79,20 +76,15 @@ public class FeedFetcher {
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && parserResult.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == parserResult.getFeed().getLastPublishedDate().getTime()) {
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
Feed feed = parserResult.getFeed();
feed.setLastModifiedHeader(result.getLastModifiedSince());
feed.setEtagHeader(FeedUtils.truncate(result.getETag(), 255));
feed.setLastContentHash(hash);
return new FeedFetcherResult(parserResult.getFeed(), parserResult.getEntries(), parserResult.getTitle(),
result.getUrlAfterRedirect(), result.getDuration());
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getDuration());
}
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
@@ -106,13 +98,8 @@ public class FeedFetcher {
return null;
}
@Value
public static class FeedFetcherResult {
Feed feed;
List<FeedEntry> entries;
String title;
String urlAfterRedirect;
long fetchDuration;
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, long fetchDuration) {
}
}

View File

@@ -1,263 +0,0 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.Value;
/**
* Parses raw xml as a Feed object
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedParser {
private static final String ATOM_10_URI = "http://www.w3.org/2005/Atom";
private static final Namespace ATOM_10_NS = Namespace.getNamespace(ATOM_10_URI);
private static final Date START = new Date(86400000);
private static final Date END = new Date(1000L * Integer.MAX_VALUE - 86400000);
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedException {
try {
Charset encoding = FeedUtils.guessEncoding(xml);
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
}
xmlString = FeedUtils.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed rss = new SyndFeedInput().build(source);
handleForeignMarkup(rss);
String title = rss.getTitle();
Feed feed = new Feed();
feed.setUrl(feedUrl);
feed.setLink(rss.getLink());
List<FeedEntry> entries = new ArrayList<>();
for (SyndEntry item : rss.getEntries()) {
FeedEntry entry = new FeedEntry();
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
entry.setGuid(FeedUtils.truncate(guid, 2048));
entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feedUrl), 2048));
// if link is empty but guid is used as url
if (StringUtils.isBlank(entry.getUrl()) && StringUtils.startsWith(entry.getGuid(), "http")) {
entry.setUrl(entry.getGuid());
}
FeedEntryContent content = new FeedEntryContent();
content.setContent(getContent(item));
content.setCategories(FeedUtils
.truncate(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")), 4096));
content.setTitle(getTitle(item));
content.setAuthor(StringUtils.trimToNull(item.getAuthor()));
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure != null) {
content.setEnclosureUrl(FeedUtils.truncate(enclosure.getUrl(), 2048));
content.setEnclosureType(enclosure.getType());
}
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module != null) {
Media media = getMedia(module);
if (media != null) {
content.setMediaDescription(media.getDescription());
content.setMediaThumbnailUrl(FeedUtils.truncate(media.getThumbnailUrl(), 2048));
content.setMediaThumbnailWidth(media.getThumbnailWidth());
content.setMediaThumbnailHeight(media.getThumbnailHeight());
}
}
entry.setContent(content);
entries.add(entry);
}
Date lastEntryDate = null;
Date publishedDate = validateDate(rss.getPublishedDate(), false);
if (!entries.isEmpty()) {
List<Long> sortedTimestamps = FeedUtils.getSortedTimestamps(entries);
Long timestamp = sortedTimestamps.get(0);
lastEntryDate = new Date(timestamp);
publishedDate = (publishedDate == null || publishedDate.before(lastEntryDate)) ? lastEntryDate : publishedDate;
}
feed.setLastPublishedDate(publishedDate);
feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
feed.setLastEntryDate(lastEntryDate);
return new FeedParserResult(feed, entries, title);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private Date getEntryUpdateDate(SyndEntry item) {
Date date = item.getUpdatedDate();
if (date == null) {
date = item.getPublishedDate();
}
if (date == null) {
date = new Date();
}
return date;
}
private Date validateDate(Date date, boolean nullToNow) {
Date now = new Date();
if (date == null) {
return nullToNow ? now : null;
}
if (date.before(START) || date.after(END)) {
return now;
}
if (date.after(now)) {
return now;
}
return date;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media getMedia(MediaEntryModule module) {
Media media = getMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = getMedia(group.getMetadata());
}
return media;
}
private Media getMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
Media media = new Media();
media.setDescription(metadata.getDescription());
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
media.setThumbnailWidth(thumbnail.getWidth());
media.setThumbnailHeight(thumbnail.getHeight());
if (thumbnail.getUrl() != null) {
media.setThumbnailUrl(thumbnail.getUrl().toString());
}
}
if (media.isEmpty()) {
return null;
}
return media;
}
@Data
private static class Media {
private String description;
private String thumbnailUrl;
private Integer thumbnailWidth;
private Integer thumbnailHeight;
public boolean isEmpty() {
return description == null && thumbnailUrl == null;
}
}
@Value
public static class FeedParserResult {
Feed feed;
List<FeedEntry> entries;
String title;
}
}

View File

@@ -1,6 +1,7 @@
package com.commafeed.backend.feed;
import java.util.Date;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
@@ -11,8 +12,6 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.time.DateUtils;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
@@ -156,7 +155,7 @@ public class FeedRefreshEngine implements Managed {
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.getFeed(), r.getEntries()), databaseUpdaterExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
@@ -166,12 +165,12 @@ public class FeedRefreshEngine implements Managed {
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Date lastLoginThreshold = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad())
? DateUtils.addDays(new Date(), -30)
Instant lastLoginThreshold = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad())
? Instant.now().minus(Duration.ofDays(30))
: null;
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Date nextUpdateDate = DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes());
Instant nextUpdateDate = Instant.now().plus(Duration.ofMinutes(config.getApplicationSettings().getRefreshIntervalMinutes()));
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});

View File

@@ -1,11 +1,10 @@
package com.commafeed.backend.feed;
import java.util.Date;
import org.apache.commons.lang3.time.DateUtils;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
@@ -22,62 +21,59 @@ public class FeedRefreshIntervalCalculator {
this.refreshIntervalMinutes = config.getApplicationSettings().getRefreshIntervalMinutes();
}
public Date onFetchSuccess(Feed feed) {
Date defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(feed, defaultRefreshInterval) : defaultRefreshInterval;
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval) {
Instant defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(publishedDate, averageEntryInterval, defaultRefreshInterval)
: defaultRefreshInterval;
}
public Date onFeedNotModified(Feed feed) {
Date defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(feed, defaultRefreshInterval) : defaultRefreshInterval;
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval);
}
public Date onFetchError(Feed feed) {
int errorCount = feed.getErrorCount();
public Instant onFetchError(int errorCount) {
int retriesBeforeDisable = 3;
if (errorCount < retriesBeforeDisable || !heavyLoad) {
return getDefaultRefreshInterval();
}
int disabledHours = Math.min(24 * 7, errorCount - retriesBeforeDisable + 1);
return DateUtils.addHours(new Date(), disabledHours);
return Instant.now().plus(Duration.ofHours(disabledHours));
}
private Date getDefaultRefreshInterval() {
return DateUtils.addMinutes(new Date(), refreshIntervalMinutes);
private Instant getDefaultRefreshInterval() {
return Instant.now().plus(Duration.ofMinutes(refreshIntervalMinutes));
}
private Date computeRefreshIntervalForHeavyLoad(Feed feed, Date defaultRefreshInterval) {
Date now = new Date();
Date publishedDate = feed.getLastEntryDate();
Long averageEntryInterval = feed.getAverageEntryInterval();
private Instant computeRefreshIntervalForHeavyLoad(Instant publishedDate, Long averageEntryInterval, Instant defaultRefreshInterval) {
Instant now = Instant.now();
if (publishedDate == null) {
// feed with no entries, recheck in 24 hours
return DateUtils.addHours(now, 24);
} else if (publishedDate.before(DateUtils.addMonths(now, -1))) {
return now.plus(Duration.ofHours(24));
} else if (ChronoUnit.DAYS.between(publishedDate, now) >= 30) {
// older than a month, recheck in 24 hours
return DateUtils.addHours(now, 24);
} else if (publishedDate.before(DateUtils.addDays(now, -14))) {
return now.plus(Duration.ofHours(24));
} else if (ChronoUnit.DAYS.between(publishedDate, now) >= 14) {
// older than two weeks, recheck in 12 hours
return DateUtils.addHours(now, 12);
} else if (publishedDate.before(DateUtils.addDays(now, -7))) {
return now.plus(Duration.ofHours(12));
} else if (ChronoUnit.DAYS.between(publishedDate, now) >= 7) {
// older than a week, recheck in 6 hours
return DateUtils.addHours(now, 6);
return now.plus(Duration.ofHours(6));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
// not more than 6 hours
long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor);
long date = Math.min(now.plus(Duration.ofHours(6)).toEpochMilli(), now.toEpochMilli() + averageEntryInterval / factor);
// not less than default refresh interval
date = Math.max(defaultRefreshInterval.getTime(), date);
date = Math.max(defaultRefreshInterval.toEpochMilli(), date);
return new Date(date);
return Instant.ofEpochMilli(date);
} else {
// unknown case, recheck in 24 hours
return DateUtils.addHours(now, 24);
return now.plus(Duration.ofHours(24));
}
}

View File

@@ -1,10 +1,11 @@
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
@@ -16,9 +17,9 @@ import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryService;
@@ -27,7 +28,6 @@ import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.AllArgsConstructor;
@@ -38,7 +38,7 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater implements Managed {
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
@@ -72,7 +72,7 @@ public class FeedRefreshUpdater implements Managed {
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) {
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
@@ -82,8 +82,8 @@ public class FeedRefreshUpdater implements Managed {
// lock on content, make sure we are not updating the same entry
// twice at the same time
FeedEntryContent content = entry.getContent();
String key2 = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent() + content.getTitle()));
Content content = entry.content();
String key2 = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
@@ -116,19 +116,19 @@ public class FeedRefreshUpdater implements Managed {
return new AddEntryResult(processed, inserted);
}
public boolean update(Feed feed, List<FeedEntry> entries) {
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
boolean insertedAtLeastOneEntry = false;
if (!entries.isEmpty()) {
List<String> lastEntries = cache.getLastEntries(feed);
Set<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>();
List<FeedSubscription> subscriptions = null;
for (FeedEntry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(feed, entry);
for (Entry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
log.debug("cache miss for {}", entry.url());
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
@@ -138,7 +138,7 @@ public class FeedRefreshUpdater implements Managed {
entryCacheMiss.mark();
} else {
log.debug("cache hit for {}", entry.getUrl());
log.debug("cache hit for {}", entry.url());
entryCacheHit.mark();
}
@@ -160,7 +160,7 @@ public class FeedRefreshUpdater implements Managed {
if (!processed) {
// requeue asap
feed.setDisabledUntil(new Date(0));
feed.setDisabledUntil(Instant.EPOCH);
}
if (insertedAtLeastOneEntry) {

View File

@@ -1,5 +1,7 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -11,16 +13,15 @@ import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database ({@link FeedRefreshUpdater} does that)
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
@@ -44,32 +45,41 @@ public class FeedRefreshWorker {
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult feedFetcherResult = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<FeedEntry> entries = feedFetcherResult.getEntries();
List<Entry> entries = result.feed().entries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
String urlAfterRedirect = feedFetcherResult.getUrlAfterRedirect();
Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxEntriesAgeDays > 0) {
Instant threshold = Instant.now().minus(Duration.ofDays(maxEntriesAgeDays));
entries = entries.stream().filter(entry -> entry.updated().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(feedFetcherResult.getFeed().getLink());
feed.setLastModifiedHeader(feedFetcherResult.getFeed().getLastModifiedHeader());
feed.setEtagHeader(feedFetcherResult.getFeed().getEtagHeader());
feed.setLastContentHash(feedFetcherResult.getFeed().getLastContentHash());
feed.setLastPublishedDate(feedFetcherResult.getFeed().getLastPublishedDate());
feed.setAverageEntryInterval(feedFetcherResult.getFeed().getAverageEntryInterval());
feed.setLastEntryDate(feedFetcherResult.getFeed().getLastEntryDate());
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(feedFetcherResult.getFeed()));
feed.setDisabledUntil(
refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), result.feed().averageEntryInterval()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
@@ -77,7 +87,7 @@ public class FeedRefreshWorker {
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed));
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
@@ -93,7 +103,7 @@ public class FeedRefreshWorker {
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed));
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
@@ -101,10 +111,7 @@ public class FeedRefreshWorker {
}
}
@Value
public static class FeedRefreshWorkerResult {
Feed feed;
List<FeedEntry> entries;
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}

View File

@@ -2,20 +2,12 @@ package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.ahocorasick.trie.Trie.TrieBuilder;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@@ -29,8 +21,6 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import com.google.gwt.i18n.client.HasDirection.Direction;
import com.google.gwt.i18n.shared.BidiUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import lombok.extern.slf4j.Slf4j;
@@ -50,70 +40,6 @@ public class FeedUtils {
return string;
}
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public static Charset guessEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Detect encoding by analyzing characters in the array
*/
public static Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
public static String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
TrieBuilder builder = Trie.builder();
builder.ignoreOverlaps();
for (String key : HtmlEntities.HTML_ENTITIES) {
builder.addKeyword(key);
}
Trie trie = builder.build();
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
@@ -122,6 +48,10 @@ public class FeedUtils {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
@@ -163,25 +93,6 @@ public class FeedUtils {
return normalized;
}
/**
* Extract the declared encoding from the xml
*/
public static String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
public static boolean isRTL(FeedEntry entry) {
String text = entry.getContent().getContent();
@@ -202,52 +113,6 @@ public class FeedUtils {
return direction == Direction.RTL;
}
public static String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
public static Long averageTimeBetweenEntries(List<FeedEntry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
List<Long> timestamps = getSortedTimestamps(entries);
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < timestamps.size() - 1; i++) {
long diff = Math.abs(timestamps.get(i) - timestamps.get(i + 1));
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static List<Long> getSortedTimestamps(List<FeedEntry> entries) {
return entries.stream().map(t -> t.getUpdated().getTime()).sorted(Collections.reverseOrder()).toList();
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
@@ -256,8 +121,8 @@ public class FeedUtils {
}
/**
*
* @param url
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
@@ -265,32 +130,18 @@ public class FeedUtils {
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String url, String feedLink, String feedUrl) {
url = StringUtils.trimToNull(StringUtils.normalizeSpace(url));
if (url == null || url.startsWith("http")) {
return url;
}
String baseUrl = (feedLink == null || isRelative(feedLink)) ? feedUrl : feedLink;
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return url;
return null;
}
String result;
try {
result = new URL(new URL(baseUrl), url).toString();
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : " + e.getMessage(), e);
result = url;
return null;
}
return result;
}
public static boolean isRelative(final String url) {
// the regex means "start with 'scheme://'"
return url.startsWith("/") || url.startsWith("#") || !url.matches("^\\w+\\:\\/\\/.*");
}
public static String getFaviconUrl(FeedSubscription subscription) {

View File

@@ -0,0 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import jakarta.inject.Singleton;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

View File

@@ -0,0 +1,63 @@
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
import jakarta.inject.Singleton;
@Singleton
class FeedCleaner {
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618/1885506
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
}

View File

@@ -0,0 +1,271 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::updated).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant updated = buildEntryUpdateDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, updated, content));
}
entries.sort(Comparator.comparing(Entry::updated).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryUpdateDate(SyndEntry item) {
Date date = item.getUpdatedDate();
if (date == null) {
date = item.getPublishedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).updated().toEpochMilli() - entries.get(i + 1).updated().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
}

View File

@@ -0,0 +1,20 @@
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant updated, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}

View File

@@ -1,10 +1,13 @@
package com.commafeed.backend.feed;
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
public class HtmlEntities {
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;

View File

@@ -1,12 +1,10 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@@ -44,20 +42,20 @@ public class Feed extends AbstractModel {
/**
* Last time we tried to fetch the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastUpdated;
@Column
private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastPublishedDate;
@Column
private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastEntryDate;
@Column
private Instant lastEntryDate;
/**
* error message while retrieving the feed
@@ -73,8 +71,8 @@ public class Feed extends AbstractModel {
/**
* feed refresh is disabled until this date
*/
@Temporal(TemporalType.TIMESTAMP)
private Date disabledUntil;
@Column
private Instant disabledUntil;
/**
* http header returned by the feed

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
@@ -11,8 +11,6 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@@ -39,11 +37,11 @@ public class FeedEntry extends AbstractModel {
@Column(length = 2048)
private String url;
@Temporal(TemporalType.TIMESTAMP)
private Date inserted;
@Column
private Instant inserted;
@Temporal(TemporalType.TIMESTAMP)
private Date updated;
@Column
private Instant updated;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;

View File

@@ -69,13 +69,13 @@ public class FeedEntryContent extends AbstractModel {
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.append(categories, c.categories)
.build();
}

View File

@@ -1,7 +1,7 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import jakarta.persistence.Column;
@@ -10,8 +10,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import jakarta.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@@ -49,11 +47,11 @@ public class FeedEntryStatus extends AbstractModel {
@JoinColumn(nullable = false)
private User user;
@Temporal(TemporalType.TIMESTAMP)
private Date entryInserted;
@Column
private Instant entryInserted;
@Temporal(TemporalType.TIMESTAMP)
private Date entryUpdated;
@Column
private Instant entryUpdated;
public FeedEntryStatus() {

View File

@@ -1,12 +1,10 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@@ -35,15 +33,15 @@ public class User extends AbstractModel {
@Column(nullable = false)
private boolean disabled;
@Temporal(TemporalType.TIMESTAMP)
private Date lastLogin;
@Column
private Instant lastLogin;
@Temporal(TemporalType.TIMESTAMP)
private Date created;
@Column
private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Temporal(TemporalType.TIMESTAMP)
private Date recoverPasswordTokenDate;
@Column
private Instant recoverPasswordTokenDate;
}

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.time.Instant;
import java.util.List;
import com.codahale.metrics.Meter;
@@ -83,6 +83,7 @@ public class DatabaseCleaningService {
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
@@ -105,7 +106,20 @@ public class DatabaseCleaningService {
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanStatusesOlderThan(final Date olderThan) {
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.info("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;

View File

@@ -0,0 +1,174 @@
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser();
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}

View File

@@ -1,206 +1,69 @@
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import com.steadystate.css.parser.CSSOMParser;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Slf4j
@Singleton
public class FeedEntryContentService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(FeedEntryContent content, String baseUrl) {
content.setAuthor(FeedUtils.truncate(handleContent(content.getAuthor(), baseUrl, true), 128));
content.setTitle(FeedUtils.truncate(handleContent(content.getTitle(), baseUrl, true), 2048));
content.setContent(handleContent(content.getContent(), baseUrl, false));
content.setMediaDescription(handleContent(content.getMediaDescription(), baseUrl, false));
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
String contentHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent()));
content.setContentHash(contentHash);
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
String titleHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getTitle()));
content.setTitleHash(titleHash);
List<FeedEntryContent> existing = feedEntryContentDAO.findExisting(contentHash, titleHash);
Optional<FeedEntryContent> equivalentContent = existing.stream().filter(content::equivalentTo).findFirst();
if (equivalentContent.isPresent()) {
return equivalentContent.get();
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
feedEntryContentDAO.saveOrUpdate(content);
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String handleContent(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return content;
return entryContent;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser();
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.time.Instant;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
@@ -10,9 +10,10 @@ import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
@@ -37,30 +38,27 @@ public class FeedEntryService {
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
public boolean addEntry(Feed feed, Entry entry, List<FeedSubscription> subscriptions) {
String guid = FeedUtils.truncate(entry.guid(), 2048);
String guidHash = DigestUtils.sha1Hex(entry.guid());
Long existing = feedEntryDAO.findExisting(guidHash, feed);
if (existing != null) {
return false;
}
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink());
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
FeedEntry feedEntry = buildEntry(feed, entry, guid, guidHash);
feedEntryDAO.saveOrUpdate(feedEntry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), feedEntry);
} catch (FeedEntryFilteringService.FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, feedEntry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
@@ -69,8 +67,20 @@ public class FeedEntryService {
return true;
}
public void markEntry(User user, Long entryId, boolean read) {
private FeedEntry buildEntry(Feed feed, Entry e, String guid, String guidHash) {
FeedEntry entry = new FeedEntry();
entry.setGuid(guid);
entry.setGuidHash(guidHash);
entry.setUrl(FeedUtils.truncate(e.url(), 2048));
entry.setUpdated(e.updated());
entry.setInserted(Instant.now());
entry.setFeed(feed);
entry.setContent(feedEntryContentService.findOrCreate(e.content(), feed.getLink()));
return entry;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
@@ -107,7 +117,7 @@ public class FeedEntryService {
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Date olderThan, Date insertedBefore,
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, false, null, null, null);
@@ -116,18 +126,18 @@ public class FeedEntryService {
cache.invalidateUserRootCategory(user);
}
public void markStarredEntries(User user, Date olderThan, Date insertedBefore) {
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Date olderThan, Date insertedBefore) {
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(s -> {
Date entryDate = s.getEntry().getUpdated();
return olderThan == null || entryDate == null || entryDate.before(olderThan);
Instant entryDate = s.getEntry().getUpdated();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Date insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.before(insertedBefore);
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));

View File

@@ -1,7 +1,7 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.Date;
import java.time.Instant;
import java.util.Set;
import org.apache.commons.codec.digest.DigestUtils;
@@ -37,14 +37,15 @@ public class FeedService {
}
public synchronized Feed findOrCreate(String url) {
String normalized = FeedUtils.normalizeURL(url);
Feed feed = feedDAO.findByUrl(normalized);
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = DigestUtils.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setDisabledUntil(new Date(0));
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Instant.EPOCH);
feedDAO.saveOrUpdate(feed);
}
return feed;
@@ -54,7 +55,8 @@ public class FeedService {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.saveOrUpdate(feed);
}

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -110,8 +110,8 @@ public class FeedSubscriptionService {
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Date disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.before(new Date())) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}

View File

@@ -1,9 +1,9 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@@ -130,7 +130,7 @@ public class UserService {
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(new Date());
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);

View File

@@ -1,8 +1,7 @@
package com.commafeed.backend.service.internal;
import java.util.Date;
import org.apache.commons.lang3.time.DateUtils;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
@@ -25,9 +24,9 @@ public class PostLoginActivities {
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Date now = new Date();
Date lastLogin = user.getLastLogin();
if (lastLogin == null || lastLogin.before(DateUtils.addMinutes(now, -30))) {
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
boolean heavyLoad = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad());

View File

@@ -0,0 +1,42 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

View File

@@ -1,5 +1,7 @@
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
@@ -18,9 +20,10 @@ public class OldEntriesCleanupTask extends ScheduledTask {
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
int maxAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxAgeDays > 0) {
Instant threshold = Instant.now().minus(Duration.ofDays(maxAgeDays));
cleaner.cleanEntriesOlderThan(threshold);
}
}

View File

@@ -1,6 +1,6 @@
package com.commafeed.backend.task;
import java.util.Date;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
@@ -19,7 +19,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
@Override
public void run() {
Date threshold = config.getApplicationSettings().getUnreadThreshold();
Instant threshold = config.getApplicationSettings().getUnreadThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
@@ -27,7 +27,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
@Override
public long getInitialDelay() {
return 10;
return 15;
}
@Override

View File

@@ -21,7 +21,7 @@ public class OrphanedContentsCleanupTask extends ScheduledTask {
@Override
public long getInitialDelay() {
return 20;
return 25;
}
@Override

View File

@@ -21,7 +21,7 @@ public class OrphanedFeedsCleanupTask extends ScheduledTask {
@Override
public long getInitialDelay() {
return 15;
return 20;
}
@Override

View File

@@ -2,12 +2,16 @@ package com.commafeed.frontend.auth;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import org.glassfish.jersey.server.ContainerRequest;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.UserService;
@@ -25,7 +29,9 @@ public class SecurityCheckFactory implements Function<ContainerRequest, User> {
private static final String PREFIX = "Basic";
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
private final HttpServletRequest request;
private final Role role;
private final boolean apiKeyAllowed;
@@ -45,21 +51,15 @@ public class SecurityCheckFactory implements Function<ContainerRequest, User> {
if (roles.contains(role)) {
return user.get();
} else {
throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN)
.entity("You don't have the required role to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
throw buildWebApplicationException(Response.Status.FORBIDDEN, "You don't have the required role to access this resource.");
}
} else {
throw new WebApplicationException(Response.status(Response.Status.UNAUTHORIZED)
.entity("Credentials are required to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
throw buildWebApplicationException(Response.Status.UNAUTHORIZED, "Credentials are required to access this resource.");
}
}
Optional<User> cookieSessionLogin(SessionHelper sessionHelper) {
Optional<User> loggedInUser = sessionHelper.getLoggedInUser();
Optional<User> loggedInUser = sessionHelper.getLoggedInUserId().map(userDAO::findById);
loggedInUser.ifPresent(userService::performPostLoginActivities);
return loggedInUser;
}
@@ -93,4 +93,11 @@ public class SecurityCheckFactory implements Function<ContainerRequest, User> {
return Optional.empty();
}
private WebApplicationException buildWebApplicationException(Response.Status status, String message) {
Map<String, Object> response = new HashMap<>();
response.put("message", message);
response.put("allowRegistrations", config.getApplicationSettings().getAllowRegistrations());
return new WebApplicationException(Response.status(status).entity(response).type(MediaType.APPLICATION_JSON).build());
}
}

View File

@@ -9,6 +9,8 @@ import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractor
import org.glassfish.jersey.server.model.Parameter;
import org.glassfish.jersey.server.spi.internal.ValueParamProvider;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
@@ -21,13 +23,17 @@ import lombok.RequiredArgsConstructor;
public class SecurityCheckFactoryProvider extends AbstractValueParamProvider {
private final UserService userService;
private final UserDAO userDAO;
private final CommaFeedConfiguration config;
private final HttpServletRequest request;
@Inject
public SecurityCheckFactoryProvider(final MultivaluedParameterExtractorProvider extractorProvider, UserService userService,
HttpServletRequest request) {
public SecurityCheckFactoryProvider(final MultivaluedParameterExtractorProvider extractorProvider, UserDAO userDAO,
UserService userService, CommaFeedConfiguration config, HttpServletRequest request) {
super(() -> extractorProvider, Parameter.Source.UNKNOWN);
this.userDAO = userDAO;
this.userService = userService;
this.config = config;
this.request = request;
}
@@ -44,18 +50,22 @@ public class SecurityCheckFactoryProvider extends AbstractValueParamProvider {
return null;
}
return new SecurityCheckFactory(userService, request, securityCheck.value(), securityCheck.apiKeyAllowed());
return new SecurityCheckFactory(userDAO, userService, config, request, securityCheck.value(), securityCheck.apiKeyAllowed());
}
@RequiredArgsConstructor
public static class Binder extends AbstractBinder {
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
@Override
protected void configure() {
bind(SecurityCheckFactoryProvider.class).to(ValueParamProvider.class).in(Singleton.class);
bind(userDAO).to(UserDAO.class);
bind(userService).to(UserService.class);
bind(config).to(CommaFeedConfiguration.class);
}
}

View File

@@ -1,6 +1,7 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@@ -67,10 +68,10 @@ public class Entry implements Serializable {
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED)
private Date date;
private Instant date;
@Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED)
private Date insertedDate;
private Instant insertedDate;
@Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED)
private String feedId;
@@ -165,7 +166,7 @@ public class Entry implements Serializable {
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate());
entry.setPublishedDate(getDate() == null ? null : Date.from(getDate()));
return entry;
}
}

View File

@@ -1,7 +1,7 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import java.time.Instant;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
@@ -30,10 +30,10 @@ public class Subscription implements Serializable {
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = "number")
private Date lastRefresh;
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number")
private Date nextRefresh;
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@@ -54,13 +54,12 @@ public class Subscription implements Serializable {
private int position;
@Schema(description = "date of the newest item", type = "number")
private Date newestItemTime;
private Instant newestItemTime;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
Date now = new Date();
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
@@ -73,7 +72,8 @@ public class Subscription implements Serializable {
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh((feed.getDisabledUntil() != null && feed.getDisabledUntil().before(now)) ? null : feed.getDisabledUntil());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));

View File

@@ -1,7 +1,7 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import java.time.Instant;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@@ -18,12 +18,12 @@ public class UnreadCount implements Serializable {
private long unreadCount;
@Schema(type = "number")
private Date newestItemTime;
private Instant newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Date newestItemTime) {
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;

View File

@@ -1,7 +1,7 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import java.time.Instant;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
@@ -31,10 +31,10 @@ public class UserModel implements Serializable {
private boolean enabled;
@Schema(description = "account creation date", type = "number")
private Date created;
private Instant created;
@Schema(description = "last login date", type = "number")
private Date lastLogin;
private Instant lastLogin;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;

View File

@@ -1,9 +1,9 @@
package com.commafeed.frontend.resource;
import java.io.StringWriter;
import java.time.Instant;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -131,7 +131,7 @@ public class CategoryREST {
id = ALL;
}
Date newerThanDate = newerThan == null ? null : new Date(newerThan);
Instant newerThanDate = newerThan == null ? null : Instant.ofEpochMilli(newerThan);
List<Long> excludedIds = null;
if (StringUtils.isNotEmpty(excludedSubscriptionIds)) {
@@ -242,8 +242,8 @@ public class CategoryREST {
Preconditions.checkNotNull(req);
Preconditions.checkNotNull(req.getId());
Date olderThan = req.getOlderThan() == null ? null : new Date(req.getOlderThan());
Date insertedBefore = req.getInsertedBefore() == null ? null : new Date(req.getInsertedBefore());
Instant olderThan = req.getOlderThan() == null ? null : Instant.ofEpochMilli(req.getOlderThan());
Instant insertedBefore = req.getInsertedBefore() == null ? null : Instant.ofEpochMilli(req.getInsertedBefore());
String keywords = req.getKeywords();
List<FeedEntryKeyword> entryKeywords = FeedEntryKeyword.fromQueryString(keywords);

View File

@@ -4,6 +4,7 @@ import java.io.InputStream;
import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
@@ -164,7 +165,7 @@ public class FeedREST {
boolean unreadOnly = readType == ReadingMode.unread;
Date newerThanDate = newerThan == null ? null : new Date(newerThan);
Instant newerThanDate = newerThan == null ? null : Instant.ofEpochMilli(newerThan);
FeedSubscription subscription = feedSubscriptionDAO.findById(user, Long.valueOf(id));
if (subscription != null) {
@@ -245,8 +246,8 @@ public class FeedREST {
try {
FeedFetcherResult feedFetcherResult = feedFetcher.fetch(url, true, null, null, null, null);
info = new FeedInfo();
info.setUrl(feedFetcherResult.getUrlAfterRedirect());
info.setTitle(feedFetcherResult.getTitle());
info.setUrl(feedFetcherResult.urlAfterRedirect());
info.setTitle(feedFetcherResult.feed().title());
} catch (Exception e) {
log.debug(e.getMessage(), e);
@@ -321,8 +322,8 @@ public class FeedREST {
Preconditions.checkNotNull(req);
Preconditions.checkNotNull(req.getId());
Date olderThan = req.getOlderThan() == null ? null : new Date(req.getOlderThan());
Date insertedBefore = req.getInsertedBefore() == null ? null : new Date(req.getInsertedBefore());
Instant olderThan = req.getOlderThan() == null ? null : Instant.ofEpochMilli(req.getOlderThan());
Instant insertedBefore = req.getInsertedBefore() == null ? null : Instant.ofEpochMilli(req.getInsertedBefore());
String keywords = req.getKeywords();
List<FeedEntryKeyword> entryKeywords = FeedEntryKeyword.fromQueryString(keywords);
@@ -379,7 +380,7 @@ public class FeedREST {
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.MONTH, 1);
builder.expires(calendar.getTime());
builder.lastModified(CommaFeedApplication.STARTUP_TIME);
builder.lastModified(Date.from(CommaFeedApplication.STARTUP_TIME));
return builder.build();
}

View File

@@ -1,14 +1,14 @@
package com.commafeed.frontend.resource;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.Date;
import java.util.Optional;
import java.util.UUID;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.hc.core5.net.URIBuilder;
import com.codahale.metrics.annotation.Timed;
@@ -153,7 +153,7 @@ public class UserREST {
s.setShowRead(settings.isShowRead());
s.setScrollMarks(settings.isScrollMarks());
s.setCustomCss(settings.getCustomCss());
s.setCustomJs(settings.getCustomJs());
s.setCustomJs(CommaFeedApplication.USERNAME_DEMO.equals(user.getName()) ? "" : settings.getCustomJs());
s.setLanguage(settings.getLanguage());
s.setScrollSpeed(settings.getScrollSpeed());
s.setAlwaysScrollToEntry(settings.isAlwaysScrollToEntry());
@@ -207,7 +207,7 @@ public class UserREST {
return Response.status(Status.FORBIDDEN).build();
}
Optional<User> login = userService.login(user.getEmail(), request.getCurrentPassword());
Optional<User> login = userService.login(user.getName(), request.getCurrentPassword());
if (login.isEmpty()) {
throw new BadRequestException("invalid password");
}
@@ -282,7 +282,7 @@ public class UserREST {
try {
user.setRecoverPasswordToken(DigestUtils.sha1Hex(UUID.randomUUID().toString()));
user.setRecoverPasswordTokenDate(new Date());
user.setRecoverPasswordTokenDate(Instant.now());
userDAO.saveOrUpdate(user);
mailService.sendMail(user, "Password recovery", buildEmailContent(user));
return Response.ok().build();
@@ -325,7 +325,7 @@ public class UserREST {
if (user.getRecoverPasswordToken() == null || !user.getRecoverPasswordToken().equals(token)) {
return Response.status(Status.UNAUTHORIZED).entity("Invalid token.").build();
}
if (user.getRecoverPasswordTokenDate().before(DateUtils.addDays(new Date(), -2))) {
if (ChronoUnit.DAYS.between(user.getRecoverPasswordTokenDate(), Instant.now()) >= 2) {
return Response.status(Status.UNAUTHORIZED).entity("token expired.").build();
}

View File

@@ -5,7 +5,6 @@ import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -189,7 +188,7 @@ public class FeverREST {
if (params.containsKey("mark") && params.containsKey("id") && params.containsKey("as")) {
long id = Long.parseLong(params.get("id"));
String before = params.get("before");
Date insertedBefore = before == null ? null : Date.from(Instant.ofEpochSecond(Long.parseLong(before)));
Instant insertedBefore = before == null ? null : Instant.ofEpochSecond(Long.parseLong(before));
mark(user, params.get("mark"), id, params.get("as"), insertedBefore);
}
@@ -206,7 +205,7 @@ public class FeverREST {
.map(Feed::getLastUpdated)
.filter(Objects::nonNull)
.max(Comparator.naturalOrder())
.map(d -> d.toInstant().getEpochSecond())
.map(d -> d.getEpochSecond())
.orElse(0L);
}
@@ -242,7 +241,7 @@ public class FeverREST {
f.setUrl(s.getFeed().getUrl());
f.setSiteUrl(s.getFeed().getLink());
f.setSpark(false);
f.setLastUpdatedOnTime(s.getFeed().getLastUpdated() == null ? 0 : s.getFeed().getLastUpdated().toInstant().getEpochSecond());
f.setLastUpdatedOnTime(s.getFeed().getLastUpdated() == null ? 0 : s.getFeed().getLastUpdated().getEpochSecond());
return f;
}).toList();
}
@@ -291,7 +290,7 @@ public class FeverREST {
i.setUrl(s.getEntry().getUrl());
i.setSaved(s.isStarred());
i.setRead(s.isRead());
i.setCreatedOnTime(s.getEntryUpdated().toInstant().getEpochSecond());
i.setCreatedOnTime(s.getEntryUpdated().getEpochSecond());
return i;
}
@@ -306,7 +305,7 @@ public class FeverREST {
}).toList();
}
private void mark(User user, String source, long id, String action, Date insertedBefore) {
private void mark(User user, String source, long id, String action, Instant insertedBefore) {
if ("item".equals(source)) {
if ("read".equals(action) || "unread".equals(action)) {
feedEntryService.markEntry(user, id, "read".equals(action));

View File

@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.Optional;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@@ -20,13 +21,16 @@ abstract class AbstractCustomCodeServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserSettingsDAO userSettingsDAO;
@Override
protected final void doGet(final HttpServletRequest req, HttpServletResponse resp) throws IOException {
resp.setContentType(getMimeType());
final Optional<User> user = new SessionHelper(req).getLoggedInUser();
SessionHelper sessionHelper = new SessionHelper(req);
Optional<Long> userId = sessionHelper.getLoggedInUserId();
final Optional<User> user = unitOfWork.call(() -> userId.map(userDAO::findById));
if (user.isEmpty()) {
return;
}

Some files were not shown because too many files have changed in this diff Show More