Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

split client and server into maven modules

Browse Source
This commit is contained in:
Athou
2022-08-13 10:34:59 +02:00
parent 4c4868a2b6
commit ac7b6eeb21
277 changed files with 645 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
commafeed-server/.gitignore vendored Normal file
View File

@@ -0,0 +1,39 @@
# config file
config.yml
# build directory
target
target-ide
# database files
database
# log files
log
# jetty sessions
sessions
# node
node
node_modules
# bower
src/main/app/lib
# Eclipse files
.project
.classpath
.settings
.factorypath
.checkstyle
# IntelliJ Idea files
.idea
*.iml
# Sublime
*.sublime*
# Macs
*.DS_Store

131
commafeed-server/config.dev.yml Normal file
View File

@@ -0,0 +1,131 @@
# CommaFeed settings
# ------------------
app:
# url used to access commafeed
publicUrl: http://localhost:8082/
# wether to allow user registrations
allowRegistrations: true
# create a demo account the first time the app starts
createDemoAccount: false
# put your google analytics tracking code here
googleAnalyticsTrackingCode:
# put your google server key (used for youtube favicon fetching)
googleAuthKey:
# number of http threads
backgroundThreads: 3
# number of database updating threads
databaseUpdateThreads: 1
# settings for sending emails (password recovery)
smtpHost: localhost
smtpPort: 25
smtpTls: false
smtpUserName: user
smtpPassword: pass
# Graphite Metric settings
# Allows those who use Graphite to have CommaFeed send metrics for graphing (time in seconds)
graphiteEnabled: false
graphitePrefix: "test.commafeed"
graphiteHost: "localhost"
graphitePort: 2003
graphiteInterval: 60
# wether this commafeed instance has a lot of feeds to refresh
# leave this to false in almost all cases
heavyLoad: false
# minimum amount of time commafeed will wait before refreshing the same feed
refreshIntervalMinutes: 5
# wether to enable pubsub
# probably not needed if refreshIntervalMinutes is low
pubsubhubbub: false
# if enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser
# useful if commafeed is usually accessed through a restricting proxy
imageProxyEnabled: false
# database query timeout (in milliseconds), 0 to disable
queryTimeout: 0
# time to keep unread statuses (in days), 0 to disable
keepStatusDays: 0
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# cache service to use, possible values are 'noop' and 'redis'
cache: noop
# announcement string displayed on the main page
announcement:
# user-agent string that will be used by the http client, leave empty for the default one
userAgent:
# Database connection
# -------------------
# for MySQL
# driverClass is com.mysql.jdbc.Driver
# url is jdbc:mysql://localhost/commafeed?autoReconnect=true&failOverReadOnly=false&maxReconnects=20&rewriteBatchedStatements=true
#
# for PostgreSQL
# driverClass is org.postgresql.Driver
# url is jdbc:postgresql://localhost:5432/commafeed
#
# for Microsoft SQL Server
# driverClass is net.sourceforge.jtds.jdbc.Driver
# url is jdbc:jtds:sqlserver://localhost:1433/commafeed;instance=<instanceName, remove if not needed>
database:
driverClass: org.h2.Driver
url: jdbc:h2:./target/example
user: sa
password: sa
properties:
charSet: UTF-8
validationQuery: "/* CommaFeed Health Check */ SELECT 1"
server:
applicationConnectors:
- type: http
port: 8083
adminConnectors:
- type: http
port: 8084
logging:
level: INFO
loggers:
com.commafeed: DEBUG
liquibase: INFO
org.hibernate.SQL: INFO # or ALL for sql debugging
org.hibernate.engine.internal.StatisticalLoggingSessionEventListener: WARN
appenders:
- type: console
- type: file
currentLogFilename: log/commafeed.log
threshold: ALL
archive: true
archivedLogFilenamePattern: log/commafeed-%d.log
archivedFileCount: 5
timeZone: UTC
# Redis pool configuration
# (only used if app.cache is 'redis')
# -----------------------------------
redis:
host: localhost
port: 6379
password:
timeout: 2000
database: 0
maxTotal: 500

135
commafeed-server/config.yml.example Normal file
View File

@@ -0,0 +1,135 @@
# CommaFeed settings
# ------------------
app:
# url used to access commafeed
publicUrl: http://localhost:8082/
# whether to allow user registrations
allowRegistrations: false
# create a demo account the first time the app starts
createDemoAccount: false
# put your google analytics tracking code here
googleAnalyticsTrackingCode:
# put your google server key (used for youtube favicon fetching)
googleAuthKey:
# number of http threads
backgroundThreads: 3
# number of database updating threads
databaseUpdateThreads: 1
# settings for sending emails (password recovery)
smtpHost:
smtpPort:
smtpTls: false
smtpUserName:
smtpPassword:
smtpFromAddress:
# Graphite Metric settings
# Allows those who use Graphite to have CommaFeed send metrics for graphing (time in seconds)
graphiteEnabled: false
graphitePrefix: "test.commafeed"
graphiteHost: "localhost"
graphitePort: 2003
graphiteInterval: 60
# wether this commafeed instance has a lot of feeds to refresh
# leave this to false in almost all cases
heavyLoad: false
# minimum amount of time commafeed will wait before refreshing the same feed
refreshIntervalMinutes: 5
# wether to enable pubsub
# probably not needed if refreshIntervalMinutes is low
pubsubhubbub: false
# if enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser
# useful if commafeed is usually accessed through a restricting proxy
imageProxyEnabled: false
# database query timeout (in milliseconds), 0 to disable
queryTimeout: 0
# time to keep unread statuses (in days), 0 to disable
keepStatusDays: 0
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# cache service to use, possible values are 'noop' and 'redis'
cache: noop
# announcement string displayed on the main page
announcement:
# user-agent string that will be used by the http client, leave empty for the default one
userAgent:
# Database connection
# -------------------
# for MySQL
# driverClass is com.mysql.jdbc.Driver
# url is jdbc:mysql://localhost/commafeed?autoReconnect=true&failOverReadOnly=false&maxReconnects=20&rewriteBatchedStatements=true
#
# for PostgreSQL
# driverClass is org.postgresql.Driver
# url is jdbc:postgresql://localhost:5432/commafeed
#
# for Microsoft SQL Server
# driverClass is net.sourceforge.jtds.jdbc.Driver
# url is jdbc:jtds:sqlserver://localhost:1433/commafeed;instance=<instanceName, remove if not needed>
database:
driverClass: org.h2.Driver
url: jdbc:h2:/home/commafeed/db
user: sa
password: sa
properties:
charSet: UTF-8
validationQuery: "/* CommaFeed Health Check */ SELECT 1"
minSize: 1
maxSize: 50
maxConnectionAge: 30m
server:
applicationConnectors:
- type: http
port: 8082
adminConnectors:
- type: http
port: 8084
logging:
level: WARN
loggers:
com.commafeed: INFO
liquibase: INFO
io.dropwizard.server.ServerFactory: INFO
org.hibernate.orm.deprecation: "OFF"
appenders:
- type: console
- type: file
currentLogFilename: log/commafeed.log
threshold: ALL
archive: true
archivedLogFilenamePattern: log/commafeed-%d.log
archivedFileCount: 5
timeZone: UTC
# Redis pool configuration
# (only used if app.cache is 'redis')
# -----------------------------------
redis:
host: localhost
port: 6379
password:
timeout: 2000
database: 0
maxTotal: 500

365
commafeed-server/dev/EclipseCodeFormatter.xml Normal file
View File

@@ -0,0 +1,365 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<profiles version="18">
<profile kind="CodeFormatterProfile" name="CommaFeed" version="18">
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.align_with_spaces" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_logical_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_method_body_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_additive_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_shift_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_lambda_body" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_parameters" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_loops" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_relational_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.text_block_indentation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_module_statements" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_conditional_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_shift_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines" value="2147483647"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_code_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_assignment_operator" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_not_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_arguments" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_relational_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_tag_description" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_string_concatenation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_logical_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_additive_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="tab"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_string_concatenation" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
</profile>
</profiles>

119
commafeed-server/dev/checkstyle.xml Normal file
View File

@@ -0,0 +1,119 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker">
<property name="charset" value="UTF-8" />
<property name="fileExtensions" value="java" />
<module name="TreeWalker">
<property name="tabWidth" value="4" />
<!-- Checks for Naming Conventions. -->
<!-- See http://checkstyle.sf.net/config_naming.html -->
<module name="CatchParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$" />
</module>
<module name="ConstantName">
<property name="format" value="^log|[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$" />
</module>
<module name="ClassTypeParameterName" />
<module name="InterfaceTypeParameterName" />
<module name="LambdaParameterName" />
<module name="LocalFinalVariableName" />
<module name="LocalVariableName" />
<module name="MemberName" />
<module name="MethodName" />
<module name="MethodTypeParameterName" />
<module name="PackageName">
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$" />
</module>
<module name="ParameterName" />
<module name="StaticVariableName" />
<module name="TypeName" />
<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
<module name="AvoidStarImport" />
<module name="AvoidStaticImport" />
<module name="IllegalImport" />
<module name="ImportOrder">
<property name="groups" value="/^java\./,javax,org,com" />
<property name="ordered" value="true" />
<property name="separated" value="true" />
</module>
<module name="RedundantImport" />
<module name="UnusedImports" />
<!-- Modifier Checks -->
<!-- See http://checkstyle.sf.net/config_modifier.html -->
<module name="ModifierOrder" />
<module name="RedundantModifier">
<property name="tokens" value="METHOD_DEF, VARIABLE_DEF, ANNOTATION_FIELD_DEF, INTERFACE_DEF, CLASS_DEF, ENUM_DEF, RESOURCE" />
</module>
<!-- Checks for blocks. You know, those {}'s -->
<!-- See http://checkstyle.sf.net/config_blocks.html -->
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="ignore|ignored" />
<message key="catch.block.empty"
value="Empty catch block. You can use the name 'ignore' or 'ignored' for the exception variable if you really want an empty catch block, but you should strongly consider at the very least logging something." />
</module>
<module name="LeftCurly" />
<module name="NeedBraces" />
<module name="RightCurly" />
<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
<module name="DeclarationOrder" />
<module name="DefaultComesLast" />
<module name="EmptyStatement" />
<module name="EqualsHashCode" />
<module name="ExplicitInitialization" />
<module name="FallThrough" />
<module name="IllegalInstantiation">
<property name="classes"
value="java.lang.Boolean, java.lang.Byte, java.lang.Character, java.lang.Double, java.lang.Float, java.lang.Integer, java.lang.Long, java.lang.Short" />
</module>
<module name="IllegalType" />
<module name="ModifiedControlVariable">
<property name="skipEnhancedForLoopVariable" value="true" />
</module>
<module name="MissingSwitchDefault" />
<module name="MultipleVariableDeclarations" />
<module name="NoFinalizer" />
<module name="OneStatementPerLine" />
<module name="OverloadMethodsDeclarationOrder" />
<module name="PackageDeclaration" />
<module name="SimplifyBooleanExpression" />
<module name="SimplifyBooleanReturn" />
<module name="StringLiteralEquality" />
<module name="UnnecessaryParentheses" />
<!-- Checks for class design -->
<!-- See http://checkstyle.sf.net/config_design.html -->
<module name="InnerTypeLast" />
<module name="OneTopLevelClass" />
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="ArrayTypeStyle" />
<module name="OuterTypeFilename" />
<module name="UpperEll" />
<!-- Whitespace checks. -->
<!-- See http://checkstyle.sourceforge.net/config_whitespace.html -->
<module name="MethodParamPad" />
<module name="NoLineWrap" />
<module name="NoWhitespaceBefore" />
<module name="ParenPad" />
<module name="RegexpSinglelineJava">
<property name="format" value="^\t* +\t*\S" />
<property name="message" value="Line has leading space characters; indentation should be performed with tabs only." />
<property name="ignoreComments" value="true" />
</module>
<module name="WhitespaceAround" />
</module>
</module>

19
commafeed-server/docker-compose.dev.yml Normal file
View File

@@ -0,0 +1,19 @@
version: "3.1"
services:
mysql:
image: mariadb
environment:
- MYSQL_ROOT_PASSWORD=root
- MYSQL_DATABASE=commafeed
ports:
- 3306:3306
postgresql:
image: postgres
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: root
POSTGRES_DB: commafeed
ports:
- 5432:5432

494
commafeed-server/pom.xml Normal file
View File

@@ -0,0 +1,494 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.commafeed</groupId>
<artifactId>commafeed</artifactId>
<version>${revision}</version>
</parent>
<artifactId>commafeed-server</artifactId>
<name>CommaFeed Server</name>
<properties>
<guice.version>5.1.0</guice.version>
<querydsl.version>4.2.1</querydsl.version>
<rome.version>1.18.0</rome.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-dependencies</artifactId>
<version>2.1.1</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<finalName>commafeed</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.10.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.22.2</version>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>2.1.13</version>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<generateGitPropertiesFile>false</generateGitPropertiesFile>
<failOnNoGitDirectory>false</failOnNoGitDirectory>
<failOnUnableToExtractRepoInfo>false</failOnUnableToExtractRepoInfo>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<dependencies>
<dependency>
<groupId>org.kordamp.shade</groupId>
<artifactId>maven-shade-ext-transformers</artifactId>
<version>1.4.0</version>
</dependency>
</dependencies>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.commafeed.CommaFeedApplication</mainClass>
</transformer>
<transformer implementation="org.kordamp.shade.resources.PropertiesFileTransformer">
<paths>
<path>rome.properties</path>
</paths>
<mergeStrategy>append</mergeStrategy>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.github.kongchen</groupId>
<artifactId>swagger-maven-plugin</artifactId>
<version>3.1.7</version>
<?m2e ignore?>
<configuration>
<apiSources>
<apiSource>
<locations>
<location>com.commafeed.frontend.resource</location>
<location>com.commafeed.frontend.model</location>
<location>com.commafeed.frontend.model.request</location>
</locations>
<swaggerDirectory>target/swagger</swaggerDirectory>
<basePath>/rest</basePath>
<info>
<title>CommaFeed</title>
<version>${project.version}</version>
</info>
<typesToSkip>
<typeToSkip>com.commafeed.backend.model.User</typeToSkip>
</typesToSkip>
</apiSource>
</apiSources>
</configuration>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>generate</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.2.2</version>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>validate</id>
<phase>validate</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<consoleOutput>true</consoleOutput>
<failsOnError>true</failsOnError>
<linkXRef>false</linkXRef>
<sourceDirectories>
<sourceDirectory>${project.build.sourceDirectory}</sourceDirectory>
</sourceDirectories>
<testSourceDirectories>
<testSourceDirectory>${project.build.testSourceDirectory}</testSourceDirectory>
</testSourceDirectories>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<configLocation>dev/checkstyle.xml</configLocation>
</configuration>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<version>1.27.0</version>
<?m2e ignore?>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<encoding>UTF-8</encoding>
<lineEndings>WINDOWS</lineEndings>
<java>
<eclipse>
<file>${project.basedir}/dev/EclipseCodeFormatter.xml</file>
</eclipse>
</java>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.commafeed</groupId>
<artifactId>commafeed-client</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>${guice.version}</version>
</dependency>
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-core</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-hibernate</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-assets</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-forms</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-graphite</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-json</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard.modules</groupId>
<artifactId>dropwizard-web</artifactId>
<version>1.5.0</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-afterburner</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
<version>1.5.22</version>
</dependency>
<dependency>
<groupId>com.querydsl</groupId>
<artifactId>querydsl-apt</artifactId>
<version>${querydsl.version}</version>
<scope>provided</scope>
<classifier>hibernate</classifier>
</dependency>
<dependency>
<groupId>com.querydsl</groupId>
<artifactId>querydsl-jpa</artifactId>
<version>${querydsl.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-jexl</artifactId>
<version>2.1.1</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.passay</groupId>
<artifactId>passay</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>com.rometools</groupId>
<artifactId>rome</artifactId>
<version>${rome.version}</version>
</dependency>
<dependency>
<groupId>com.rometools</groupId>
<artifactId>rome-modules</artifactId>
<version>${rome.version}</version>
</dependency>
<dependency>
<groupId>com.rometools</groupId>
<artifactId>rome-opml</artifactId>
<version>${rome.version}</version>
</dependency>
<dependency>
<groupId>org.ahocorasick</groupId>
<artifactId>ahocorasick</artifactId>
<version>0.6.3</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.3</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>70.1</version>
</dependency>
<dependency>
<groupId>net.sourceforge.cssparser</groupId>
<artifactId>cssparser</artifactId>
<version>0.9.29</version>
</dependency>
<dependency>
<groupId>edu.uci.ics</groupId>
<artifactId>crawler4j</artifactId>
<version>3.5</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.gwt</groupId>
<artifactId>gwt-servlet</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>io.github.hakky54</groupId>
<artifactId>sslcontext-kickstart</artifactId>
<version>7.2.0</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-youtube</artifactId>
<version>v3-rev139-1.20.0</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.28</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.4.1</version>
</dependency>
<dependency>
<groupId>net.sourceforge.jtds</groupId>
<artifactId>jtds</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-junit-jupiter</artifactId>
<version>5.13.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.dropwizard</groupId>
<artifactId>dropwizard-testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

182
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java Normal file
View File

@@ -0,0 +1,182 @@
package com.commafeed;
import java.io.IOException;
import java.util.Date;
import java.util.EnumSet;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.servlet.DispatcherType;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import org.hibernate.cfg.AvailableSettings;
import com.codahale.metrics.json.MetricsModule;
import com.commafeed.backend.feed.FeedRefreshTaskGiver;
import com.commafeed.backend.feed.FeedRefreshUpdater;
import com.commafeed.backend.feed.FeedRefreshWorker;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.StartupService;
import com.commafeed.backend.service.UserService;
import com.commafeed.backend.task.ScheduledTask;
import com.commafeed.frontend.auth.SecurityCheckFactoryProvider;
import com.commafeed.frontend.resource.AdminREST;
import com.commafeed.frontend.resource.CategoryREST;
import com.commafeed.frontend.resource.EntryREST;
import com.commafeed.frontend.resource.FeedREST;
import com.commafeed.frontend.resource.PubSubHubbubCallbackREST;
import com.commafeed.frontend.resource.ServerREST;
import com.commafeed.frontend.resource.UserREST;
import com.commafeed.frontend.servlet.AnalyticsServlet;
import com.commafeed.frontend.servlet.CustomCssServlet;
import com.commafeed.frontend.servlet.LogoutServlet;
import com.commafeed.frontend.servlet.NextUnreadServlet;
import com.commafeed.frontend.session.SessionHelperFactoryProvider;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.dropwizard.Application;
import io.dropwizard.assets.AssetsBundle;
import io.dropwizard.db.DataSourceFactory;
import io.dropwizard.forms.MultiPartBundle;
import io.dropwizard.hibernate.HibernateBundle;
import io.dropwizard.server.DefaultServerFactory;
import io.dropwizard.servlets.CacheBustingFilter;
import io.dropwizard.setup.Bootstrap;
import io.dropwizard.setup.Environment;
import io.dropwizard.web.WebBundle;
import io.dropwizard.web.conf.WebConfiguration;
public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
public static final Date STARTUP_TIME = new Date();
private HibernateBundle<CommaFeedConfiguration> hibernateBundle;
@Override
public String getName() {
return "CommaFeed";
}
@Override
public void initialize(Bootstrap<CommaFeedConfiguration> bootstrap) {
bootstrap.getObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
bootstrap.addBundle(hibernateBundle = new HibernateBundle<CommaFeedConfiguration>(AbstractModel.class, Feed.class,
FeedCategory.class, FeedEntry.class, FeedEntryContent.class, FeedEntryStatus.class, FeedEntryTag.class,
FeedSubscription.class, User.class, UserRole.class, UserSettings.class) {
@Override
public DataSourceFactory getDataSourceFactory(CommaFeedConfiguration configuration) {
DataSourceFactory factory = configuration.getDataSourceFactory();
// keep using old id generator for backward compatibility
factory.getProperties().put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
factory.getProperties().put(AvailableSettings.STATEMENT_BATCH_SIZE, "50");
factory.getProperties().put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
return factory;
}
});
bootstrap.addBundle(new WebBundle<CommaFeedConfiguration>() {
@Override
public WebConfiguration getWebConfiguration(CommaFeedConfiguration configuration) {
WebConfiguration config = new WebConfiguration();
config.getFrameOptionsHeaderFactory().setEnabled(true);
return config;
}
});
bootstrap.addBundle(new AssetsBundle("/assets/", "/", "index.html"));
bootstrap.addBundle(new MultiPartBundle());
}
@Override
public void run(CommaFeedConfiguration config, Environment environment) throws Exception {
// guice init
Injector injector = Guice.createInjector(new CommaFeedModule(hibernateBundle.getSessionFactory(), config, environment.metrics()));
// session management
environment.servlets().setSessionHandler(config.getSessionHandlerFactory().build());
// support for "@SecurityCheck User user" injection
environment.jersey().register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserService.class)));
// support for "@Context SessionHelper sessionHelper" injection
environment.jersey().register(new SessionHelperFactoryProvider.Binder());
// REST resources
environment.jersey().setUrlPattern("/rest/*");
((DefaultServerFactory) config.getServerFactory()).setJerseyRootPath("/rest/*");
environment.jersey().register(injector.getInstance(AdminREST.class));
environment.jersey().register(injector.getInstance(CategoryREST.class));
environment.jersey().register(injector.getInstance(EntryREST.class));
environment.jersey().register(injector.getInstance(FeedREST.class));
environment.jersey().register(injector.getInstance(PubSubHubbubCallbackREST.class));
environment.jersey().register(injector.getInstance(ServerREST.class));
environment.jersey().register(injector.getInstance(UserREST.class));
// Servlets
environment.servlets().addServlet("next", injector.getInstance(NextUnreadServlet.class)).addMapping("/next");
environment.servlets().addServlet("logout", injector.getInstance(LogoutServlet.class)).addMapping("/logout");
environment.servlets().addServlet("customCss", injector.getInstance(CustomCssServlet.class)).addMapping("/custom_css.css");
environment.servlets().addServlet("analytics.js", injector.getInstance(AnalyticsServlet.class)).addMapping("/analytics.js");
// Scheduled tasks
Set<ScheduledTask> tasks = injector.getInstance(Key.get(new TypeLiteral<Set<ScheduledTask>>() {
}));
ScheduledExecutorService executor = environment.lifecycle()
.scheduledExecutorService("task-scheduler", true)
.threads(tasks.size())
.build();
for (ScheduledTask task : tasks) {
task.register(executor);
}
// database init/changelogs
environment.lifecycle().manage(injector.getInstance(StartupService.class));
// background feed fetching
environment.lifecycle().manage(injector.getInstance(FeedRefreshTaskGiver.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshWorker.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshUpdater.class));
// cache configuration
// prevent caching on REST resources, except for favicons
environment.servlets().addFilter("cache-filter", new CacheBustingFilter() {
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
String path = ((HttpServletRequest) request).getRequestURI();
if (path.contains("/feed/favicon")) {
chain.doFilter(request, response);
} else {
super.doFilter(request, response, chain);
}
}
}).addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/rest/*");
}
public static void main(String[] args) throws Exception {
new CommaFeedApplication().run(args);
}
}

152
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java Normal file
View File

@@ -0,0 +1,152 @@
package com.commafeed;
import java.util.Date;
import java.util.ResourceBundle;
import javax.validation.Valid;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.apache.commons.lang3.time.DateUtils;
import com.commafeed.backend.cache.RedisPoolFactory;
import com.commafeed.frontend.session.SessionHandlerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.dropwizard.Configuration;
import io.dropwizard.db.DataSourceFactory;
import lombok.Getter;
@Getter
public class CommaFeedConfiguration extends Configuration {
public enum CacheType {
NOOP, REDIS
}
@Valid
@NotNull
@JsonProperty("database")
private final DataSourceFactory dataSourceFactory = new DataSourceFactory();
@Valid
@NotNull
@JsonProperty("redis")
private final RedisPoolFactory redisPoolFactory = new RedisPoolFactory();
@Valid
@NotNull
@JsonProperty("session")
private final SessionHandlerFactory sessionHandlerFactory = new SessionHandlerFactory();
@Valid
@NotNull
@JsonProperty("app")
private ApplicationSettings applicationSettings;
private final ResourceBundle bundle;
public CommaFeedConfiguration() {
bundle = ResourceBundle.getBundle("application");
}
public String getVersion() {
return bundle.getString("version");
}
public String getGitCommit() {
return bundle.getString("git.commit");
}
@Getter
public static class ApplicationSettings {
@NotNull
@NotBlank
@Valid
private String publicUrl;
@NotNull
@Valid
private Boolean allowRegistrations;
@NotNull
@Valid
private Boolean createDemoAccount;
private String googleAnalyticsTrackingCode;
private String googleAuthKey;
@NotNull
@Min(1)
@Valid
private Integer backgroundThreads;
@NotNull
@Min(1)
@Valid
private Integer databaseUpdateThreads;
private String smtpHost;
private int smtpPort;
private boolean smtpTls;
private String smtpUserName;
private String smtpPassword;
private String smtpFromAddress;
private boolean graphiteEnabled;
private String graphitePrefix;
private String graphiteHost;
private int graphitePort;
private int graphiteInterval;
@NotNull
@Valid
private Boolean heavyLoad;
@NotNull
@Valid
private Boolean pubsubhubbub;
@NotNull
@Valid
private Boolean imageProxyEnabled;
@NotNull
@Min(0)
@Valid
private Integer queryTimeout;
@NotNull
@Min(0)
@Valid
private Integer keepStatusDays;
@NotNull
@Min(0)
@Valid
private Integer maxFeedCapacity;
@NotNull
@Min(0)
@Valid
private Integer refreshIntervalMinutes;
@NotNull
@Valid
private CacheType cache;
@Valid
private String announcement;
private String userAgent;
public Date getUnreadThreshold() {
int keepStatusDays = getKeepStatusDays();
return keepStatusDays > 0 ? DateUtils.addDays(new Date(), -1 * keepStatusDays) : null;
}
}
}

97
commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java Normal file
View File

@@ -0,0 +1,97 @@
package com.commafeed;
import java.net.InetSocketAddress;
import java.util.concurrent.TimeUnit;
import org.hibernate.SessionFactory;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.graphite.Graphite;
import com.codahale.metrics.graphite.GraphiteReporter;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.CommaFeedConfiguration.CacheType;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.cache.NoopCacheService;
import com.commafeed.backend.cache.RedisCacheService;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.DefaultFaviconFetcher;
import com.commafeed.backend.favicon.FacebookFaviconFetcher;
import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
import com.commafeed.backend.task.DemoAccountCleanupTask;
import com.commafeed.backend.task.OldEntriesCleanupTask;
import com.commafeed.backend.task.OldStatusesCleanupTask;
import com.commafeed.backend.task.OrphanedContentsCleanupTask;
import com.commafeed.backend.task.OrphanedFeedsCleanupTask;
import com.commafeed.backend.task.ScheduledTask;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.commafeed.backend.urlprovider.InPageReferenceFeedURLProvider;
import com.commafeed.backend.urlprovider.YoutubeFeedURLProvider;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.multibindings.Multibinder;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class CommaFeedModule extends AbstractModule {
@Getter(onMethod = @__({ @Provides }))
private final SessionFactory sessionFactory;
@Getter(onMethod = @__({ @Provides }))
private final CommaFeedConfiguration config;
@Getter(onMethod = @__({ @Provides }))
private final MetricRegistry metrics;
@Override
protected void configure() {
CacheService cacheService = config.getApplicationSettings().getCache() == CacheType.NOOP ? new NoopCacheService()
: new RedisCacheService(config.getRedisPoolFactory().build());
log.info("using cache {}", cacheService.getClass());
bind(CacheService.class).toInstance(cacheService);
Multibinder<AbstractFaviconFetcher> faviconMultibinder = Multibinder.newSetBinder(binder(), AbstractFaviconFetcher.class);
faviconMultibinder.addBinding().to(YoutubeFaviconFetcher.class);
faviconMultibinder.addBinding().to(FacebookFaviconFetcher.class);
faviconMultibinder.addBinding().to(DefaultFaviconFetcher.class);
Multibinder<FeedURLProvider> urlProviderMultibinder = Multibinder.newSetBinder(binder(), FeedURLProvider.class);
urlProviderMultibinder.addBinding().to(InPageReferenceFeedURLProvider.class);
urlProviderMultibinder.addBinding().to(YoutubeFeedURLProvider.class);
Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);
taskMultibinder.addBinding().to(DemoAccountCleanupTask.class);
ApplicationSettings settings = config.getApplicationSettings();
if (settings.isGraphiteEnabled()) {
final String graphitePrefix = settings.getGraphitePrefix();
final String graphiteHost = settings.getGraphiteHost();
final int graphitePort = settings.getGraphitePort();
final int graphiteInterval = settings.getGraphiteInterval();
log.info("Graphite Metrics will be sent to host={}, port={}, prefix={}, interval={}sec", graphiteHost, graphitePort,
graphitePrefix, graphiteInterval);
final Graphite graphite = new Graphite(new InetSocketAddress(graphiteHost, graphitePort));
final GraphiteReporter reporter = GraphiteReporter.forRegistry(metrics)
.prefixedWith(graphitePrefix)
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(MetricFilter.ALL)
.build(graphite);
reporter.start(graphiteInterval, TimeUnit.SECONDS);
} else {
log.info("Graphite Metrics Disabled. Metrics will not be sent.");
}
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/ContentEncodingInterceptor.java Normal file
View File

@@ -0,0 +1,60 @@
package com.commafeed.backend;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.protocol.HttpContext;
class ContentEncodingInterceptor implements HttpResponseInterceptor {
private static final Set<String> ALLOWED_CONTENT_ENCODINGS = new HashSet<>(Arrays.asList("gzip", "x-gzip", "deflate", "identity"));
@Override
public void process(HttpResponse response, HttpContext context) throws HttpException, IOException {
if (hasContent(response)) {
Header contentEncodingHeader = response.getEntity().getContentEncoding();
if (contentEncodingHeader != null && containsUnsupportedEncodings(contentEncodingHeader)) {
overrideContentEncoding(response);
}
}
}
private boolean containsUnsupportedEncodings(Header contentEncodingHeader) {
HeaderElement[] codecs = contentEncodingHeader.getElements();
for (final HeaderElement codec : codecs) {
String codecName = codec.getName().toLowerCase(Locale.US);
if (!ALLOWED_CONTENT_ENCODINGS.contains(codecName)) {
return true;
}
}
return false;
}
private void overrideContentEncoding(HttpResponse response) {
HttpEntity wrapped = new HttpEntityWrapper(response.getEntity()) {
@Override
public Header getContentEncoding() {
return null;
}
};
response.setEntity(wrapped);
}
private boolean hasContent(HttpResponse response) {
return response.getEntity() != null && response.getEntity().getContentLength() != 0;
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/FixedSizeSortedSet.java Normal file
View File

@@ -0,0 +1,49 @@
package com.commafeed.backend;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/**
* List wrapper that sorts its elements in the order provided by given comparator and ensure a maximum capacity.
*
*
*/
public class FixedSizeSortedSet<E> {
private List<E> inner;
private final Comparator<? super E> comparator;
private final int capacity;
public FixedSizeSortedSet(int capacity, Comparator<? super E> comparator) {
this.inner = new ArrayList<E>(Math.max(0, capacity));
this.capacity = capacity < 0 ? Integer.MAX_VALUE : capacity;
this.comparator = comparator;
}
public void add(E e) {
int position = Math.abs(Collections.binarySearch(inner, e, comparator) + 1);
if (isFull()) {
if (position < inner.size()) {
inner.remove(inner.size() - 1);
inner.add(position, e);
}
} else {
inner.add(position, e);
}
}
public E last() {
return inner.get(inner.size() - 1);
}
public boolean isFull() {
return inner.size() == capacity;
}
public List<E> asList() {
return inner;
}
}

205
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java Normal file
View File

@@ -0,0 +1,205 @@
package com.commafeed.backend;
import java.io.IOException;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import com.commafeed.CommaFeedConfiguration;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*
*/
@Singleton
public class HttpGetter {
private static final String ACCEPT_LANGUAGE = "en";
private static final String PRAGMA_NO_CACHE = "No-cache";
private static final String CACHE_CONTROL_NO_CACHE = "no-cache";
private static final HttpResponseInterceptor REMOVE_INCORRECT_CONTENT_ENCODING = new ContentEncodingInterceptor();
private static final SSLFactory SSL_FACTORY = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
private String userAgent;
@Inject
public HttpGetter(CommaFeedConfiguration config) {
this.userAgent = config.getApplicationSettings().getUserAgent();
if (this.userAgent == null) {
this.userAgent = String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", config.getVersion());
}
}
public HttpResult getBinary(String url, int timeout) throws ClientProtocolException, IOException, NotModifiedException {
return getBinary(url, null, null, timeout);
}
/**
*
* @param url
* the url to retrive
* @param lastModified
* header we got last time we queried that url, or null
* @param eTag
* header we got last time we queried that url, or null
* @return
* @throws ClientProtocolException
* @throws IOException
* @throws NotModifiedException
* if the url hasn't changed since we asked for it last time
*/
public HttpResult getBinary(String url, String lastModified, String eTag, int timeout)
throws ClientProtocolException, IOException, NotModifiedException {
HttpResult result = null;
long start = System.currentTimeMillis();
CloseableHttpClient client = newClient(timeout);
CloseableHttpResponse response = null;
try {
HttpGet httpget = new HttpGet(url);
HttpClientContext context = HttpClientContext.create();
httpget.addHeader(HttpHeaders.ACCEPT_LANGUAGE, ACCEPT_LANGUAGE);
httpget.addHeader(HttpHeaders.PRAGMA, PRAGMA_NO_CACHE);
httpget.addHeader(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_NO_CACHE);
httpget.addHeader(HttpHeaders.USER_AGENT, userAgent);
if (lastModified != null) {
httpget.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
httpget.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
try {
response = client.execute(httpget, context);
int code = response.getStatusLine().getStatusCode();
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
} else if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
} catch (HttpResponseException e) {
if (e.getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
} else {
throw e;
}
}
Header lastModifiedHeader = response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
String lastModifiedHeaderValue = lastModifiedHeader == null ? null : StringUtils.trimToNull(lastModifiedHeader.getValue());
if (lastModifiedHeaderValue != null && StringUtils.equals(lastModified, lastModifiedHeaderValue)) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
Header eTagHeader = response.getFirstHeader(HttpHeaders.ETAG);
String eTagHeaderValue = eTagHeader == null ? null : StringUtils.trimToNull(eTagHeader.getValue());
if (eTag != null && StringUtils.equals(eTag, eTagHeaderValue)) {
throw new NotModifiedException("eTagHeader is the same");
}
HttpEntity entity = response.getEntity();
byte[] content = null;
String contentType = null;
if (entity != null) {
content = EntityUtils.toByteArray(entity);
if (entity.getContentType() != null) {
contentType = entity.getContentType().getValue();
}
}
String urlAfterRedirect = url;
if (context.getRequest() instanceof HttpUriRequest) {
HttpUriRequest req = (HttpUriRequest) context.getRequest();
HttpHost host = context.getTargetHost();
urlAfterRedirect = req.getURI().isAbsolute() ? req.getURI().toString() : host.toURI() + req.getURI();
}
long duration = System.currentTimeMillis() - start;
result = new HttpResult(content, contentType, lastModifiedHeaderValue, eTagHeaderValue, duration, urlAfterRedirect);
} finally {
IOUtils.closeQuietly(response);
IOUtils.closeQuietly(client);
}
return result;
}
public static CloseableHttpClient newClient(int timeout) {
HttpClientBuilder builder = HttpClients.custom();
builder.useSystemProperties();
builder.addInterceptorFirst(REMOVE_INCORRECT_CONTENT_ENCODING);
builder.disableAutomaticRetries();
builder.setSSLContext(SSL_FACTORY.getSslContext());
builder.setSSLHostnameVerifier(SSL_FACTORY.getHostnameVerifier());
RequestConfig.Builder configBuilder = RequestConfig.custom();
configBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
configBuilder.setSocketTimeout(timeout);
configBuilder.setConnectTimeout(timeout);
configBuilder.setConnectionRequestTimeout(timeout);
builder.setDefaultRequestConfig(configBuilder.build());
builder.setDefaultConnectionConfig(ConnectionConfig.custom().setCharset(Consts.ISO_8859_1).build());
return builder.build();
}
public static void main(String[] args) throws Exception {
CommaFeedConfiguration config = new CommaFeedConfiguration();
HttpGetter getter = new HttpGetter(config);
HttpResult result = getter.getBinary("https://sourceforge.net/projects/mpv-player-windows/rss", 30000);
System.out.println(new String(result.content));
}
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
public NotModifiedException(String message) {
super(message);
}
}
@Getter
@RequiredArgsConstructor
public static class HttpResult {
private final byte[] content;
private final String contentType;
private final String lastModifiedSince;
private final String eTag;
private final long duration;
private final String urlAfterRedirect;
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/cache/CacheService.java vendored Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.cache;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
public abstract class CacheService {
// feed entries for faster refresh
public abstract List<String> getLastEntries(Feed feed);
public abstract void setLastEntries(Feed feed, List<String> entries);
public String buildUniqueEntryKey(Feed feed, FeedEntry entry) {
return DigestUtils.sha1Hex(entry.getGuid() + entry.getUrl());
}
// user categories
public abstract Category getUserRootCategory(User user);
public abstract void setUserRootCategory(User user, Category category);
public abstract void invalidateUserRootCategory(User... users);
// unread count
public abstract UnreadCount getUnreadCount(FeedSubscription sub);
public abstract void setUnreadCount(FeedSubscription sub, UnreadCount count);
public abstract void invalidateUnreadCount(FeedSubscription... subs);
}

53
commafeed-server/src/main/java/com/commafeed/backend/cache/NoopCacheService.java vendored Normal file
View File

@@ -0,0 +1,53 @@
package com.commafeed.backend.cache;
import java.util.Collections;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
public class NoopCacheService extends CacheService {
@Override
public List<String> getLastEntries(Feed feed) {
return Collections.emptyList();
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
}
@Override
public UnreadCount getUnreadCount(FeedSubscription sub) {
return null;
}
@Override
public void setUnreadCount(FeedSubscription sub, UnreadCount count) {
}
@Override
public void invalidateUnreadCount(FeedSubscription... subs) {
}
@Override
public Category getUserRootCategory(User user) {
return null;
}
@Override
public void setUserRootCategory(User user, Category category) {
}
@Override
public void invalidateUserRootCategory(User... users) {
}
}

159
commafeed-server/src/main/java/com/commafeed/backend/cache/RedisCacheService.java vendored Normal file
View File

@@ -0,0 +1,159 @@
package com.commafeed.backend.cache;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.Pipeline;
@Slf4j
@RequiredArgsConstructor
public class RedisCacheService extends CacheService {
private static final ObjectMapper MAPPER = new ObjectMapper();
private final JedisPool pool;
@Override
public List<String> getLastEntries(Feed feed) {
List<String> list = new ArrayList<>();
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Set<String> members = jedis.smembers(key);
for (String member : members) {
list.add(member);
}
}
return list;
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
for (String entry : entries) {
pipe.sadd(key, entry);
}
pipe.expire(key, (int) TimeUnit.DAYS.toSeconds(7));
pipe.sync();
}
}
@Override
public Category getUserRootCategory(User user) {
Category cat = null;
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUserRootCategoryKey(user);
String json = jedis.get(key);
if (json != null) {
cat = MAPPER.readValue(json, Category.class);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return cat;
}
@Override
public void setUserRootCategory(User user, Category category) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUserRootCategoryKey(user);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
pipe.set(key, MAPPER.writeValueAsString(category));
pipe.expire(key, (int) TimeUnit.MINUTES.toSeconds(30));
pipe.sync();
} catch (JsonProcessingException e) {
log.error(e.getMessage(), e);
}
}
@Override
public UnreadCount getUnreadCount(FeedSubscription sub) {
UnreadCount count = null;
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUnreadCountKey(sub);
String json = jedis.get(key);
if (json != null) {
count = MAPPER.readValue(json, UnreadCount.class);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return count;
}
@Override
public void setUnreadCount(FeedSubscription sub, UnreadCount count) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUnreadCountKey(sub);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
pipe.set(key, MAPPER.writeValueAsString(count));
pipe.expire(key, (int) TimeUnit.MINUTES.toSeconds(30));
pipe.sync();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
@Override
public void invalidateUserRootCategory(User... users) {
try (Jedis jedis = pool.getResource()) {
Pipeline pipe = jedis.pipelined();
if (users != null) {
for (User user : users) {
String key = buildRedisUserRootCategoryKey(user);
pipe.del(key);
}
}
pipe.sync();
}
}
@Override
public void invalidateUnreadCount(FeedSubscription... subs) {
try (Jedis jedis = pool.getResource()) {
Pipeline pipe = jedis.pipelined();
if (subs != null) {
for (FeedSubscription sub : subs) {
String key = buildRedisUnreadCountKey(sub);
pipe.del(key);
}
}
pipe.sync();
}
}
private String buildRedisEntryKey(Feed feed) {
return "f:" + Models.getId(feed);
}
private String buildRedisUserRootCategoryKey(User user) {
return "c:" + Models.getId(user);
}
private String buildRedisUnreadCountKey(FeedSubscription sub) {
return "u:" + Models.getId(sub);
}
}

27
commafeed-server/src/main/java/com/commafeed/backend/cache/RedisPoolFactory.java vendored Normal file
View File

@@ -0,0 +1,27 @@
package com.commafeed.backend.cache;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Protocol;
@Getter
public class RedisPoolFactory {
private final String host = "localhost";
private final int port = Protocol.DEFAULT_PORT;
private String password;
private final int timeout = Protocol.DEFAULT_TIMEOUT;
private final int database = Protocol.DEFAULT_DATABASE;
private final int maxTotal = 500;
public JedisPool build() {
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxTotal(maxTotal);
return new JedisPool(config, host, port, timeout, StringUtils.trimToNull(password), database);
}
}

75
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java Normal file
View File

@@ -0,0 +1,75 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private QFeedCategory category = QFeedCategory.feedCategory;
@Inject
public FeedCategoryDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(category).where(category.user.eq(user)).join(category.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(category).where(category.user.eq(user), category.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate = null;
if (parent == null) {
parentPredicate = category.parent.isNull();
} else {
parentPredicate = category.parent.eq(parent);
}
return query().selectFrom(category).where(category.user.eq(user), category.name.eq(name), parentPredicate).fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate = null;
if (parent == null) {
parentPredicate = category.parent.isNull();
} else {
parentPredicate = category.parent.eq(parent);
}
return query().selectFrom(category).where(category.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).collect(Collectors.toList());
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}

62
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java Normal file
View File

@@ -0,0 +1,62 @@
package com.commafeed.backend.dao;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.QUser;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private final QFeed feed = QFeed.feed;
@Inject
public FeedDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<Feed> findNextUpdatable(int count, Date lastLoginThreshold) {
JPQLQuery<Feed> query = query().selectFrom(feed);
query.where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(new Date())));
if (lastLoginThreshold != null) {
QFeedSubscription subs = QFeedSubscription.feedSubscription;
QUser user = QUser.user;
query.join(feed.subscriptions, subs).join(subs.user, user).where(user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(feed.disabledUntil.asc()).limit(count).fetch();
}
public Feed findByUrl(String normalizedUrl) {
List<Feed> feeds = query().selectFrom(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).fetch();
Feed feed = Iterables.getFirst(feeds, null);
if (feed != null && StringUtils.equals(normalizedUrl, feed.getNormalizedUrl())) {
return feed;
}
return null;
}
public List<Feed> findByTopic(String topic) {
return query().selectFrom(feed).where(feed.pushTopicHash.eq(DigestUtils.sha1Hex(topic))).fetch();
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(feed).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(feed)).notExists()).limit(max).fetch();
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java Normal file
View File

@@ -0,0 +1,40 @@
package com.commafeed.backend.dao;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private final QFeedEntryContent content = QFeedEntryContent.feedEntryContent;
private final QFeedEntry entry = QFeedEntry.feedEntry;
@Inject
public FeedEntryContentDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(content).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).fetch();
}
public int deleteWithoutEntries(int max) {
JPQLQuery<Integer> subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id));
List<FeedEntryContent> list = query().selectFrom(content).where(subQuery.notExists()).limit(max).fetch();
int deleted = list.size();
delete(list);
return deleted;
}
}

67
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java Normal file
View File

@@ -0,0 +1,67 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private QFeedEntry entry = QFeedEntry.feedEntry;
@Inject
public FeedEntryDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public Long findExisting(String guid, Feed feed) {
return query().select(entry.id)
.from(entry)
.where(entry.guidHash.eq(DigestUtils.sha1Hex(guid)), entry.feed.eq(feed))
.limit(1)
.fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
NumberExpression<Long> count = entry.id.count();
List<Tuple> tuples = query().select(entry.feed.id, count)
.from(entry)
.groupBy(entry.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch();
return tuples.stream().map(t -> new FeedCapacity(t.get(entry.feed.id), t.get(count))).collect(Collectors.toList());
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);
}
@AllArgsConstructor
@Getter
public static class FeedCapacity {
private Long id;
private Long capacity;
}
}

277
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java Normal file
View File

@@ -0,0 +1,277 @@
package com.commafeed.backend.dao;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.builder.CompareToBuilder;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.FixedSizeSortedSet;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_DESC = new Comparator<FeedEntryStatus>() {
@Override
public int compare(FeedEntryStatus o1, FeedEntryStatus o2) {
CompareToBuilder builder = new CompareToBuilder();
builder.append(o2.getEntryUpdated(), o1.getEntryUpdated());
builder.append(o2.getId(), o1.getId());
return builder.toComparison();
}
};
private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_ASC = Ordering.from(STATUS_COMPARATOR_DESC).reverse();
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
private final QFeedEntryStatus status = QFeedEntryStatus.feedEntryStatus;
private final QFeedEntry entry = QFeedEntry.feedEntry;
private final QFeedEntryContent content = QFeedEntryContent.feedEntryContent;
private final QFeedEntryTag entryTag = QFeedEntryTag.feedEntryTag;
@Inject
public FeedEntryStatusDAO(SessionFactory sessionFactory, FeedEntryDAO feedEntryDAO, FeedEntryTagDAO feedEntryTagDAO,
CommaFeedConfiguration config) {
super(sessionFactory);
this.feedEntryDAO = feedEntryDAO;
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(status).where(status.entry.eq(entry), status.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
boolean read = unreadThreshold == null ? false : entry.getUpdated().before(unreadThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private FeedEntryStatus fetchTags(User user, FeedEntryStatus status) {
List<FeedEntryTag> tags = feedEntryTagDAO.findByEntry(user, status.getEntry());
status.setTags(tags);
return status;
}
public List<FeedEntryStatus> findStarred(User user, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue());
if (newerThan != null) {
query.where(status.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.asc) {
query.orderBy(status.entryUpdated.asc(), status.id.asc());
} else {
query.orderBy(status.entryUpdated.desc(), status.id.desc());
}
query.offset(offset).limit(limit);
setTimeout(query, config.getApplicationSettings().getQueryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
for (FeedEntryStatus status : statuses) {
status = handleStatus(user, status, status.getSubscription(), status.getEntry());
fetchTags(user, status);
}
return lazyLoadContent(includeContent, statuses);
}
private JPAQuery<FeedEntry> buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List<FeedEntryKeyword> keywords,
Date newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag) {
JPAQuery<FeedEntry> query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed()));
if (CollectionUtils.isNotEmpty(keywords)) {
query.join(entry.content, content);
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(content.content.containsIgnoreCase(keyword.getKeyword()));
or.or(content.title.containsIgnoreCase(keyword.getKeyword()));
if (keyword.getMode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
query.leftJoin(entry.statuses, status).on(status.subscription.id.eq(sub.getId()));
if (unreadOnly && tag == null) {
BooleanBuilder or = new BooleanBuilder();
or.or(status.read.isNull());
or.or(status.read.isFalse());
query.where(or);
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
if (unreadThreshold != null) {
query.where(entry.updated.goe(unreadThreshold));
}
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(entryTag.user.id.eq(user.getId()));
and.and(entryTag.name.eq(tag));
query.join(entry.tags, entryTag).on(and);
}
if (newerThan != null) {
query.where(entry.inserted.goe(newerThan));
}
if (last != null) {
if (order == ReadingOrder.desc) {
query.where(entry.updated.gt(last.getEntryUpdated()));
} else {
query.where(entry.updated.lt(last.getEntryUpdated()));
}
}
if (order != null) {
if (order == ReadingOrder.asc) {
query.orderBy(entry.updated.asc(), entry.id.asc());
} else {
query.orderBy(entry.updated.desc(), entry.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.getApplicationSettings().getQueryTimeout());
return query;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
boolean onlyIds, String tag) {
int capacity = offset + limit;
Comparator<FeedEntryStatus> comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC;
FixedSizeSortedSet<FeedEntryStatus> set = new FixedSizeSortedSet<>(capacity, comparator);
for (FeedSubscription sub : subs) {
FeedEntryStatus last = (order != null && set.isFull()) ? set.last() : null;
JPAQuery<FeedEntry> query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag);
List<Tuple> tuples = query.select(entry.id, entry.updated, status.id, entry.content.title).fetch();
for (Tuple tuple : tuples) {
Long id = tuple.get(entry.id);
Date updated = tuple.get(entry.updated);
Long statusId = tuple.get(status.id);
FeedEntryContent content = new FeedEntryContent();
content.setTitle(tuple.get(entry.content.title));
FeedEntry entry = new FeedEntry();
entry.setId(id);
entry.setUpdated(updated);
entry.setContent(content);
FeedEntryStatus status = new FeedEntryStatus();
status.setId(statusId);
status.setEntryUpdated(updated);
status.setEntry(entry);
status.setSubscription(sub);
set.add(status);
}
}
List<FeedEntryStatus> placeholders = set.asList();
int size = placeholders.size();
if (size < offset) {
return new ArrayList<>();
}
placeholders = placeholders.subList(Math.max(offset, 0), size);
List<FeedEntryStatus> statuses = null;
if (onlyIds) {
statuses = placeholders;
} else {
statuses = new ArrayList<>();
for (FeedEntryStatus placeholder : placeholders) {
Long statusId = placeholder.getId();
FeedEntry entry = feedEntryDAO.findById(placeholder.getEntry().getId());
FeedEntryStatus status = handleStatus(user, statusId == null ? null : findById(statusId), placeholder.getSubscription(),
entry);
status = fetchTags(user, status);
statuses.add(status);
}
statuses = lazyLoadContent(includeContent, statuses);
}
return statuses;
}
public UnreadCount getUnreadCount(User user, FeedSubscription subscription) {
UnreadCount uc = null;
JPAQuery<FeedEntry> query = buildQuery(user, subscription, true, null, null, -1, -1, null, null, null);
List<Tuple> tuples = query.select(entry.count(), entry.updated.max()).fetch();
for (Tuple tuple : tuples) {
Long count = tuple.get(entry.count());
Date updated = tuple.get(entry.updated.max());
uc = new UnreadCount(subscription.getId(), count, updated);
}
return uc;
}
private List<FeedEntryStatus> lazyLoadContent(boolean includeContent, List<FeedEntryStatus> results) {
if (includeContent) {
for (FeedEntryStatus status : results) {
Models.initialize(status.getSubscription().getFeed());
Models.initialize(status.getEntry().getContent());
}
}
return results;
}
public List<FeedEntryStatus> getOldStatuses(Date olderThan, int limit) {
return query().selectFrom(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).fetch();
}
}

32
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java Normal file
View File

@@ -0,0 +1,32 @@
package com.commafeed.backend.dao;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private QFeedEntryTag tag = QFeedEntryTag.feedEntryTag;
@Inject
public FeedEntryTagDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<String> findByUser(User user) {
return query().selectDistinct(tag.name).from(tag).where(tag.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(tag).where(tag.user.eq(user), tag.entry.eq(entry)).fetch();
}
}

91
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java Normal file
View File

@@ -0,0 +1,91 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private QFeedSubscription sub = QFeedSubscription.feedSubscription;
@Inject
public FeedSubscriptionDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(sub)
.where(sub.user.eq(user), sub.id.eq(id))
.leftJoin(sub.feed)
.fetchJoin()
.leftJoin(sub.category)
.fetchJoin()
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(sub).where(sub.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(sub).where(sub.user.eq(user), sub.feed.eq(feed)).fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(sub)
.where(sub.user.eq(user))
.leftJoin(sub.feed)
.fetchJoin()
.leftJoin(sub.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(sub).where(sub.user.eq(user));
if (category == null) {
query.where(sub.category.isNull());
} else {
query.where(sub.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(c -> c.getId()).collect(Collectors.toSet());
return findAll(user).stream()
.filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId()))
.collect(Collectors.toList());
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(s -> initRelations(s));
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java Normal file
View File

@@ -0,0 +1,60 @@
package com.commafeed.backend.dao;
import java.util.Collection;
import org.hibernate.SessionFactory;
import org.hibernate.annotations.QueryHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import io.dropwizard.hibernate.AbstractDAO;
public abstract class GenericDAO<T extends AbstractModel> extends AbstractDAO<T> {
private JPAQueryFactory factory;
protected GenericDAO(SessionFactory sessionFactory) {
super(sessionFactory);
this.factory = new JPAQueryFactory(() -> currentSession());
}
protected JPAQueryFactory query() {
return factory;
}
public void saveOrUpdate(T model) {
persist(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(m -> persist(m));
}
public void update(T model) {
currentSession().merge(model);
}
public T findById(Long id) {
return get(id);
}
public void delete(T object) {
if (object != null) {
currentSession().delete(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(o -> delete(o));
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, int timeoutMs) {
if (timeoutMs > 0) {
query.setHint(QueryHints.TIMEOUT_JPA, timeoutMs);
}
}
}

69
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java Normal file
View File

@@ -0,0 +1,69 @@
package com.commafeed.backend.dao;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.context.internal.ManagedSessionContext;
public class UnitOfWork {
public static void run(SessionFactory sessionFactory, SessionRunner sessionRunner) {
call(sessionFactory, () -> {
sessionRunner.runInSession();
return null;
});
}
public static <T> T call(SessionFactory sessionFactory, SessionRunnerReturningValue<T> sessionRunner) {
final Session session = sessionFactory.openSession();
if (ManagedSessionContext.hasBind(sessionFactory)) {
throw new IllegalStateException("Already in a unit of work!");
}
T t = null;
try {
ManagedSessionContext.bind(session);
session.beginTransaction();
try {
t = sessionRunner.runInSession();
commitTransaction(session);
} catch (Exception e) {
rollbackTransaction(session);
UnitOfWork.<RuntimeException> rethrow(e);
}
} finally {
session.close();
ManagedSessionContext.unbind(sessionFactory);
}
return t;
}
private static void rollbackTransaction(Session session) {
final Transaction txn = session.getTransaction();
if (txn != null && txn.isActive()) {
txn.rollback();
}
}
private static void commitTransaction(Session session) {
final Transaction txn = session.getTransaction();
if (txn != null && txn.isActive()) {
txn.commit();
}
}
@SuppressWarnings("unchecked")
private static <E extends Exception> void rethrow(Exception e) throws E {
throw (E) e;
}
@FunctionalInterface
public interface SessionRunner {
void runInSession();
}
@FunctionalInterface
public interface SessionRunnerReturningValue<T> {
T runInSession();
}
}

36
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java Normal file
View File

@@ -0,0 +1,36 @@
package com.commafeed.backend.dao;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
@Singleton
public class UserDAO extends GenericDAO<User> {
private QUser user = QUser.user;
@Inject
public UserDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public User findByName(String name) {
return query().selectFrom(user).where(user.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(user).where(user.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(user).where(user.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().selectFrom(user).fetchCount();
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java Normal file
View File

@@ -0,0 +1,38 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private QUserRole role = QUserRole.userRole;
@Inject
public UserRoleDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<UserRole> findAll() {
return query().selectFrom(role).leftJoin(role.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(role).where(role.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(r -> r.getRole()).collect(Collectors.toSet());
}
}

25
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java Normal file
View File

@@ -0,0 +1,25 @@
package com.commafeed.backend.dao;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private QUserSettings settings = QUserSettings.userSettings;
@Inject
public UserSettingsDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public UserSettings findByUser(User user) {
return query().selectFrom(settings).where(settings.user.eq(user)).fetchFirst();
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java Normal file
View File

@@ -0,0 +1,60 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
protected static final int TIMEOUT = 4000;
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
@RequiredArgsConstructor
@Getter
public static class Favicon {
private final byte[] icon;
private final String mediaType;
}
}

132
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java Normal file
View File

@@ -0,0 +1,132 @@
package com.commafeed.backend.favicon;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.getBinary(url, TIMEOUT);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc = null;
try {
HttpResult result = getter.getBinary(url, TIMEOUT);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.get(0).attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes = null;
String contentType = null;
try {
HttpResult result = getter.getBinary(href, TIMEOUT);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

79
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java Normal file
View File

@@ -0,0 +1,79 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.getBinary(iconUrl, TIMEOUT);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {
URI uri = null;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = URLEncodedUtils.parse(uri, StandardCharsets.UTF_8);
for (NameValuePair param : params) {
if ("id".equals(param.getName())) {
return param.getValue();
}
}
return null;
}
}

100
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java Normal file
View File

@@ -0,0 +1,100 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.youtube.YouTube;
import com.google.api.services.youtube.model.Channel;
import com.google.api.services.youtube.model.ChannelListResponse;
import com.google.api.services.youtube.model.Thumbnail;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
private final CommaFeedConfiguration config;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey();
if (googleAuthKey == null) {
log.debug("no google auth key configured");
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = URLEncodedUtils.parse(url.substring(url.indexOf("?") + 1), StandardCharsets.UTF_8);
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
if (!userId.isPresent() && !channelId.isPresent()) {
return null;
}
YouTube youtube = new YouTube.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance(),
new HttpRequestInitializer() {
@Override
public void initialize(HttpRequest request) throws IOException {
}
}).setApplicationName("CommaFeed").build();
YouTube.Channels.List list = youtube.channels().list("snippet");
list.setKey(googleAuthKey);
if (userId.isPresent()) {
list.setForUsername(userId.get().getValue());
} else {
list.setId(channelId.get().getValue());
}
log.debug("contacting youtube api");
ChannelListResponse response = list.execute();
if (response.getItems().isEmpty()) {
log.debug("youtube api returned no items");
return null;
}
Channel channel = response.getItems().get(0);
Thumbnail thumbnail = channel.getSnippet().getThumbnails().getDefault();
log.debug("fetching favicon");
HttpResult iconResult = getter.getBinary(thumbnail.getUrl(), TIMEOUT);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE;
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

96
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java Normal file
View File

@@ -0,0 +1,96 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.util.Date;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders;
public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate,
String lastContentHash) throws FeedException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
int timeout = 20000;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag, timeout);
byte[] content = result.getContent();
try {
fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, StringUtils.newStringUtf8(result.getContent()));
if (org.apache.commons.lang3.StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.getBinary(extractedUrl, lastModified, eTag, timeout);
content = result.getContent();
fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw e;
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
String hash = DigestUtils.sha1Hex(content);
if (lastContentHash != null && hash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified");
}
if (lastPublishedDate != null && fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified");
}
Feed feed = fetchedFeed.getFeed();
feed.setLastModifiedHeader(result.getLastModifiedSince());
feed.setEtagHeader(FeedUtils.truncate(result.getETag(), 255));
feed.setLastContentHash(hash);
fetchedFeed.setFetchDuration(result.getDuration());
fetchedFeed.setUrlAfterRedirect(result.getUrlAfterRedirect());
return fetchedFeed;
}
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
}

276
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedParser.java Normal file
View File

@@ -0,0 +1,276 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedParser {
private static final String ATOM_10_URI = "http://www.w3.org/2005/Atom";
private static final Namespace ATOM_10_NS = Namespace.getNamespace(ATOM_10_URI);
private static final Date START = new Date(86400000);
private static final Date END = new Date(1000L * Integer.MAX_VALUE - 86400000);
public FetchedFeed parse(String feedUrl, byte[] xml) throws FeedException {
FetchedFeed fetchedFeed = new FetchedFeed();
Feed feed = fetchedFeed.getFeed();
List<FeedEntry> entries = fetchedFeed.getEntries();
try {
Charset encoding = FeedUtils.guessEncoding(xml);
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
}
xmlString = FeedUtils.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed rss = new SyndFeedInput().build(source);
handleForeignMarkup(rss);
fetchedFeed.setTitle(rss.getTitle());
feed.setPushHub(findHub(rss));
feed.setPushTopic(findSelf(rss));
feed.setUrl(feedUrl);
feed.setLink(rss.getLink());
List<SyndEntry> items = rss.getEntries();
for (SyndEntry item : items) {
FeedEntry entry = new FeedEntry();
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
entry.setGuid(FeedUtils.truncate(guid, 2048));
entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feedUrl), 2048));
// if link is empty but guid is used as url
if (StringUtils.isBlank(entry.getUrl()) && StringUtils.startsWith(entry.getGuid(), "http")) {
entry.setUrl(entry.getGuid());
}
FeedEntryContent content = new FeedEntryContent();
content.setContent(getContent(item));
content.setCategories(FeedUtils
.truncate(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")), 4096));
content.setTitle(getTitle(item));
content.setAuthor(StringUtils.trimToNull(item.getAuthor()));
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure != null) {
content.setEnclosureUrl(FeedUtils.truncate(enclosure.getUrl(), 2048));
content.setEnclosureType(enclosure.getType());
}
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module != null) {
Media media = getMedia(module);
if (media != null) {
content.setMediaDescription(media.getDescription());
content.setMediaThumbnailUrl(FeedUtils.truncate(media.getThumbnailUrl(), 2048));
content.setMediaThumbnailWidth(media.getThumbnailWidth());
content.setMediaThumbnailHeight(media.getThumbnailHeight());
}
}
entry.setContent(content);
entries.add(entry);
}
Date lastEntryDate = null;
Date publishedDate = validateDate(rss.getPublishedDate(), false);
if (!entries.isEmpty()) {
List<Long> sortedTimestamps = FeedUtils.getSortedTimestamps(entries);
Long timestamp = sortedTimestamps.get(0);
lastEntryDate = new Date(timestamp);
publishedDate = (publishedDate == null || publishedDate.before(lastEntryDate)) ? lastEntryDate : publishedDate;
}
feed.setLastPublishedDate(publishedDate);
feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
feed.setLastEntryDate(lastEntryDate);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
return fetchedFeed;
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private Date getEntryUpdateDate(SyndEntry item) {
Date date = item.getUpdatedDate();
if (date == null) {
date = item.getPublishedDate();
}
if (date == null) {
date = new Date();
}
return date;
}
private Date validateDate(Date date, boolean nullToNow) {
Date now = new Date();
if (date == null) {
return nullToNow ? now : null;
}
if (date.before(START) || date.after(END)) {
return now;
}
if (date.after(now)) {
return now;
}
return date;
}
private String getContent(SyndEntry item) {
String content = null;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media getMedia(MediaEntryModule module) {
Media media = getMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = getMedia(group.getMetadata());
}
return media;
}
private Media getMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
Media media = new Media();
media.setDescription(metadata.getDescription());
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
media.setThumbnailWidth(thumbnail.getWidth());
media.setThumbnailHeight(thumbnail.getHeight());
if (thumbnail.getUrl() != null) {
media.setThumbnailUrl(thumbnail.getUrl().toString());
}
}
if (media.isEmpty()) {
return null;
}
return media;
}
private String findHub(SyndFeed feed) {
for (SyndLink l : feed.getLinks()) {
if ("hub".equalsIgnoreCase(l.getRel())) {
log.debug("found hub {} for feed {}", l.getHref(), feed.getLink());
return l.getHref();
}
}
return null;
}
private String findSelf(SyndFeed feed) {
for (SyndLink l : feed.getLinks()) {
if ("self".equalsIgnoreCase(l.getRel())) {
log.debug("found self {} for feed {}", l.getHref(), feed.getLink());
return l.getHref();
}
}
return null;
}
@Data
private static class Media {
private String description;
private String thumbnailUrl;
private Integer thumbnailWidth;
private Integer thumbnailHeight;
public boolean isEmpty() {
return description == null && thumbnailUrl == null;
}
}
}

160
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedQueues.java Normal file
View File

@@ -0,0 +1,160 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
@Singleton
public class FeedQueues {
private SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final CommaFeedConfiguration config;
private Queue<FeedRefreshContext> addQueue = new ConcurrentLinkedQueue<>();
private Queue<FeedRefreshContext> takeQueue = new ConcurrentLinkedQueue<>();
private Queue<Feed> giveBackQueue = new ConcurrentLinkedQueue<>();
private Meter refill;
@Inject
public FeedQueues(SessionFactory sessionFactory, FeedDAO feedDAO, CommaFeedConfiguration config, MetricRegistry metrics) {
this.sessionFactory = sessionFactory;
this.config = config;
this.feedDAO = feedDAO;
refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
metrics.register(MetricRegistry.name(getClass(), "addQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return addQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "takeQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return takeQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "giveBackQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return giveBackQueue.size();
}
});
}
/**
* take a feed from the refresh queue
*/
public synchronized FeedRefreshContext take() {
FeedRefreshContext context = takeQueue.poll();
if (context == null) {
refill();
context = takeQueue.poll();
}
return context;
}
/**
* add a feed to the refresh queue
*/
public void add(Feed feed, boolean urgent) {
int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
if (feed.getLastUpdated() == null || feed.getLastUpdated().before(DateUtils.addMinutes(new Date(), -1 * refreshInterval))) {
boolean alreadyQueued = addQueue.stream().anyMatch(c -> c.getFeed().getId().equals(feed.getId()));
if (!alreadyQueued) {
addQueue.add(new FeedRefreshContext(feed, urgent));
}
}
}
/**
* refills the refresh queue and empties the giveBack queue while at it
*/
private void refill() {
refill.mark();
List<FeedRefreshContext> contexts = new ArrayList<>();
int batchSize = Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
// add feeds we got from the add() method
int addQueueSize = addQueue.size();
for (int i = 0; i < Math.min(batchSize, addQueueSize); i++) {
contexts.add(addQueue.poll());
}
// add feeds that are up to refresh from the database
int count = batchSize - contexts.size();
if (count > 0) {
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findNextUpdatable(count, getLastLoginThreshold()));
for (Feed feed : feeds) {
contexts.add(new FeedRefreshContext(feed, false));
}
}
// set the disabledDate as we use it in feedDAO to decide what to refresh next. We also use a map to remove
// duplicates.
Map<Long, FeedRefreshContext> map = new LinkedHashMap<>();
for (FeedRefreshContext context : contexts) {
Feed feed = context.getFeed();
feed.setDisabledUntil(DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes()));
map.put(feed.getId(), context);
}
// refill the queue
takeQueue.addAll(map.values());
// add feeds from the giveBack queue to the map, overriding duplicates
int giveBackQueueSize = giveBackQueue.size();
for (int i = 0; i < giveBackQueueSize; i++) {
Feed feed = giveBackQueue.poll();
map.put(feed.getId(), new FeedRefreshContext(feed, false));
}
// update all feeds in the database
List<Feed> feeds = map.values().stream().map(c -> c.getFeed()).collect(Collectors.toList());
UnitOfWork.run(sessionFactory, () -> feedDAO.saveOrUpdate(feeds));
}
/**
* give a feed back, updating it to the database during the next refill()
*/
public void giveBack(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
giveBackQueue.add(feed);
}
private Date getLastLoginThreshold() {
if (config.getApplicationSettings().getHeavyLoad()) {
return DateUtils.addDays(new Date(), -30);
} else {
return null;
}
}
}

22
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshContext.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.backend.feed;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FeedRefreshContext {
private Feed feed;
private List<FeedEntry> entries;
private boolean urgent;
public FeedRefreshContext(Feed feed, boolean isUrgent) {
this.feed = feed;
this.urgent = isUrgent;
}
}

99
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshExecutor.java Normal file
View File

@@ -0,0 +1,99 @@
package com.commafeed.backend.feed;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import lombok.extern.slf4j.Slf4j;
/**
* Wraps a {@link ThreadPoolExecutor} instance. Blocks when queue is full instead of rejecting the task. Allow priority queueing by using
* {@link Task} instead of {@link Runnable}
*
*/
@Slf4j
public class FeedRefreshExecutor {
private String poolName;
private ThreadPoolExecutor pool;
private LinkedBlockingDeque<Runnable> queue;
public FeedRefreshExecutor(final String poolName, int threads, int queueCapacity, MetricRegistry metrics) {
log.info("Creating pool {} with {} threads", poolName, threads);
this.poolName = poolName;
pool = new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS, queue = new LinkedBlockingDeque<Runnable>(queueCapacity) {
private static final long serialVersionUID = 1L;
@Override
public boolean offer(Runnable r) {
Task task = (Task) r;
if (task.isUrgent()) {
return offerFirst(r);
} else {
return offerLast(r);
}
}
}) {
@Override
protected void afterExecute(Runnable r, Throwable t) {
if (t != null) {
log.error("thread from pool {} threw a runtime exception", poolName, t);
}
}
};
pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
log.debug("{} thread queue full, waiting...", poolName);
try {
Task task = (Task) r;
if (task.isUrgent()) {
queue.putFirst(r);
} else {
queue.put(r);
}
} catch (InterruptedException e1) {
log.error(poolName + " interrupted while waiting for queue.", e1);
}
}
});
metrics.register(MetricRegistry.name(getClass(), poolName, "active"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return pool.getActiveCount();
}
});
metrics.register(MetricRegistry.name(getClass(), poolName, "pending"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return queue.size();
}
});
}
public void execute(Task task) {
pool.execute(task);
}
public void shutdown() {
pool.shutdownNow();
while (!pool.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("{} interrupted while waiting for threads to finish.", poolName);
}
}
}
public interface Task extends Runnable {
boolean isUrgent();
}
}

85
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java Normal file
View File

@@ -0,0 +1,85 @@
package com.commafeed.backend.feed;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
/**
* Infinite loop fetching feeds from @FeedQueues and queuing them to the {@link FeedRefreshWorker} pool.
*
*/
@Slf4j
@Singleton
public class FeedRefreshTaskGiver implements Managed {
private final FeedQueues queues;
private final FeedRefreshWorker worker;
private final ExecutorService executor;
private final Meter feedRefreshed;
private final Meter threadWaited;
@Inject
public FeedRefreshTaskGiver(FeedQueues queues, FeedDAO feedDAO, FeedRefreshWorker worker, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.queues = queues;
this.worker = worker;
executor = Executors.newFixedThreadPool(1);
feedRefreshed = metrics.meter(MetricRegistry.name(getClass(), "feedRefreshed"));
threadWaited = metrics.meter(MetricRegistry.name(getClass(), "threadWaited"));
}
@Override
public void stop() {
log.info("shutting down feed refresh task giver");
executor.shutdownNow();
while (!executor.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("interrupted while waiting for threads to finish.");
}
}
}
@Override
public void start() {
log.info("starting feed refresh task giver");
executor.execute(new Runnable() {
@Override
public void run() {
while (!executor.isShutdown()) {
try {
FeedRefreshContext context = queues.take();
if (context != null) {
feedRefreshed.mark();
worker.updateFeed(context);
} else {
log.debug("nothing to do, sleeping for 15s");
threadWaited.mark();
try {
Thread.sleep(15000);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
}
});
}
}

226
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java Normal file
View File

@@ -0,0 +1,226 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedUpdateService;
import com.commafeed.backend.service.PubSubService;
import com.google.common.util.concurrent.Striped;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshUpdater implements Managed {
private final SessionFactory sessionFactory;
private final FeedUpdateService feedUpdateService;
private final PubSubService pubSubService;
private final FeedQueues queues;
private final CommaFeedConfiguration config;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final CacheService cache;
private final FeedRefreshExecutor pool;
private final Striped<Lock> locks;
private final Meter entryCacheMiss;
private final Meter entryCacheHit;
private final Meter feedUpdated;
private final Meter entryInserted;
@Inject
public FeedRefreshUpdater(SessionFactory sessionFactory, FeedUpdateService feedUpdateService, PubSubService pubSubService,
FeedQueues queues, CommaFeedConfiguration config, MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO,
CacheService cache) {
this.sessionFactory = sessionFactory;
this.feedUpdateService = feedUpdateService;
this.pubSubService = pubSubService;
this.queues = queues;
this.config = config;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.cache = cache;
ApplicationSettings settings = config.getApplicationSettings();
int threads = Math.max(settings.getDatabaseUpdateThreads(), 1);
pool = new FeedRefreshExecutor("feed-refresh-updater", threads, Math.min(50 * threads, 1000), metrics);
locks = Striped.lazyWeakLock(threads * 100000);
entryCacheMiss = metrics.meter(MetricRegistry.name(getClass(), "entryCacheMiss"));
entryCacheHit = metrics.meter(MetricRegistry.name(getClass(), "entryCacheHit"));
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
@Override
public void start() throws Exception {
}
@Override
public void stop() throws Exception {
log.info("shutting down feed refresh updater");
pool.shutdown();
}
public void updateFeed(FeedRefreshContext context) {
pool.execute(new EntryTask(context));
}
private boolean addEntry(final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) {
boolean success = false;
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty("" + feed.getId());
// lock on content, make sure we are not updating the same entry
// twice at the same time
FeedEntryContent content = entry.getContent();
String key2 = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent() + content.getTitle()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
boolean inserted = UnitOfWork.call(sessionFactory, () -> feedUpdateService.addEntry(feed, entry, subscriptions));
if (inserted) {
entryInserted.mark();
}
success = true;
} else {
log.error("lock timeout for " + feed.getUrl() + " - " + key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for " + feed.getUrl() + " : " + e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return success;
}
private void handlePubSub(final Feed feed) {
if (feed.getPushHub() != null && feed.getPushTopic() != null) {
Date lastPing = feed.getPushLastPing();
Date now = new Date();
if (lastPing == null || lastPing.before(DateUtils.addDays(now, -3))) {
new Thread() {
@Override
public void run() {
try {
// make sure the feed has been updated in the database so that the
// callback works
Thread.sleep(30000);
} catch (InterruptedException e1) {
// do nothing
}
pubSubService.subscribe(feed);
}
}.start();
}
}
}
private class EntryTask implements Task {
private final FeedRefreshContext context;
public EntryTask(FeedRefreshContext context) {
this.context = context;
}
@Override
public void run() {
boolean ok = true;
final Feed feed = context.getFeed();
List<FeedEntry> entries = context.getEntries();
if (entries.isEmpty()) {
feed.setMessage("Feed has no entries");
} else {
List<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>();
List<FeedSubscription> subscriptions = null;
for (FeedEntry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(feed, entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
if (subscriptions == null) {
subscriptions = UnitOfWork.call(sessionFactory, () -> feedSubscriptionDAO.findByFeed(feed));
}
ok &= addEntry(feed, entry, subscriptions);
entryCacheMiss.mark();
} else {
log.debug("cache hit for {}", entry.getUrl());
entryCacheHit.mark();
}
currentEntries.add(cacheKey);
}
cache.setLastEntries(feed, currentEntries);
if (subscriptions == null) {
feed.setMessage("No new entries found");
} else if (!subscriptions.isEmpty()) {
List<User> users = subscriptions.stream().map(FeedSubscription::getUser).collect(Collectors.toList());
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
}
}
if (config.getApplicationSettings().getPubsubhubbub()) {
handlePubSub(feed);
}
if (!ok) {
// requeue asap
feed.setDisabledUntil(new Date(0));
}
feedUpdated.mark();
queues.giveBack(feed);
}
@Override
public boolean isUrgent() {
return context.isUrgent();
}
}
}

167
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java Normal file
View File

@@ -0,0 +1,167 @@
package com.commafeed.backend.feed;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and handles its outcome
*
*/
@Slf4j
@Singleton
public class FeedRefreshWorker implements Managed {
private final FeedRefreshUpdater feedRefreshUpdater;
private final FeedFetcher fetcher;
private final FeedQueues queues;
private final CommaFeedConfiguration config;
private final FeedRefreshExecutor pool;
@Inject
public FeedRefreshWorker(FeedRefreshUpdater feedRefreshUpdater, FeedFetcher fetcher, FeedQueues queues, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.feedRefreshUpdater = feedRefreshUpdater;
this.fetcher = fetcher;
this.config = config;
this.queues = queues;
int threads = config.getApplicationSettings().getBackgroundThreads();
pool = new FeedRefreshExecutor("feed-refresh-worker", threads, Math.min(20 * threads, 1000), metrics);
}
@Override
public void start() throws Exception {
}
@Override
public void stop() throws Exception {
pool.shutdown();
}
public void updateFeed(FeedRefreshContext context) {
pool.execute(new FeedTask(context));
}
private void update(FeedRefreshContext context) {
Feed feed = context.getFeed();
int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval);
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FetchedFeed fetchedFeed = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<FeedEntry> entries = fetchedFeed.getEntries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).collect(Collectors.toList());
}
if (config.getApplicationSettings().getHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(),
fetchedFeed.getFeed().getAverageEntryInterval(), disabledUntil);
}
String urlAfterRedirect = fetchedFeed.getUrlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(fetchedFeed.getFeed().getLink());
feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader());
feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
feed.setLastContentHash(fetchedFeed.getFeed().getLastContentHash());
feed.setLastPublishedDate(fetchedFeed.getFeed().getLastPublishedDate());
feed.setAverageEntryInterval(fetchedFeed.getFeed().getAverageEntryInterval());
feed.setLastEntryDate(fetchedFeed.getFeed().getLastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(disabledUntil);
handlePubSub(feed, fetchedFeed.getFeed());
context.setEntries(entries);
feedRefreshUpdater.updateFeed(context);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
if (config.getApplicationSettings().getHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil);
}
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(disabledUntil);
queues.giveBack(feed);
} catch (Exception e) {
String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
log.debug(e.getClass().getName() + " " + message, e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage(message);
feed.setDisabledUntil(FeedUtils.buildDisabledUntil(feed.getErrorCount()));
queues.giveBack(feed);
}
}
private void handlePubSub(Feed feed, Feed fetchedFeed) {
String hub = fetchedFeed.getPushHub();
String topic = fetchedFeed.getPushTopic();
if (hub != null && topic != null) {
if (hub.contains("hubbub.api.typepad.com")) {
// that hub does not exist anymore
return;
}
if (topic.startsWith("www.")) {
topic = "http://" + topic;
} else if (topic.startsWith("feed://")) {
topic = "http://" + topic.substring(7);
} else if (!topic.startsWith("http")) {
topic = "http://" + topic;
}
log.debug("feed {} has pubsub info: {}", feed.getUrl(), topic);
feed.setPushHub(hub);
feed.setPushTopic(topic);
feed.setPushTopicHash(DigestUtils.sha1Hex(topic));
}
}
private class FeedTask implements Task {
private final FeedRefreshContext context;
public FeedTask(FeedRefreshContext context) {
this.context = context;
}
@Override
public void run() {
update(context);
}
@Override
public boolean isUrgent() {
return context.isUrgent();
}
}
}

556
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java Normal file
View File

@@ -0,0 +1,556 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.ahocorasick.trie.Trie.TrieBuilder;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.jsoup.select.Elements;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import com.google.gwt.i18n.client.HasDirection.Direction;
import com.google.gwt.i18n.shared.BidiUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.steadystate.css.parser.CSSOMParser;
import edu.uci.ics.crawler4j.url.URLCanonicalizer;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
private static final Safelist WHITELIST = buildWhiteList();
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
private static synchronized Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public static Charset guessEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Detect encoding by analyzing characters in the array
*/
public static Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
public static String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
TrieBuilder builder = Trie.builder();
builder.ignoreOverlaps();
for (String key : HtmlEntities.HTML_ENTITIES) {
builder.addKeyword(key);
}
Trie trie = builder.build();
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source.substring(prevIndex, matchIndex));
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
String normalized = URLCanonicalizer.getCanonicalURL(url);
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
/**
* Extract the declared encoding from the xml
*/
public static String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10, pi.length());
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
public static String handleContent(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
public static String escapeIFrameCss(String orig) {
String rule = "";
CSSOMParser parser = new CSSOMParser();
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
public static String escapeImgCss(String orig) {
String rule = "";
CSSOMParser parser = new CSSOMParser();
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
public static boolean isRTL(FeedEntry entry) {
String text = entry.getContent().getContent();
if (StringUtils.isBlank(text)) {
text = entry.getContent().getTitle();
}
if (StringUtils.isBlank(text)) {
return false;
}
text = Jsoup.parse(text).text();
if (StringUtils.isBlank(text)) {
return false;
}
Direction direction = BidiUtils.get().estimateDirection(text);
return direction == Direction.RTL;
}
public static String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
/**
* When there was an error fetching the feed
*
*/
public static Date buildDisabledUntil(int errorCount) {
Date now = new Date();
int retriesBeforeDisable = 3;
if (errorCount >= retriesBeforeDisable) {
int disabledHours = errorCount - retriesBeforeDisable + 1;
disabledHours = Math.min(24 * 7, disabledHours);
return DateUtils.addHours(now, disabledHours);
}
return now;
}
/**
* When the feed was refreshed successfully
*/
public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval, Date defaultRefreshInterval) {
Date now = new Date();
if (publishedDate == null) {
// feed with no entries, recheck in 24 hours
return DateUtils.addHours(now, 24);
} else if (publishedDate.before(DateUtils.addMonths(now, -1))) {
// older than a month, recheck in 24 hours
return DateUtils.addHours(now, 24);
} else if (publishedDate.before(DateUtils.addDays(now, -14))) {
// older than two weeks, recheck in 12 hours
return DateUtils.addHours(now, 12);
} else if (publishedDate.before(DateUtils.addDays(now, -7))) {
// older than a week, recheck in 6 hours
return DateUtils.addHours(now, 6);
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
// not more than 6 hours
long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor);
// not less than default refresh interval
date = Math.max(defaultRefreshInterval.getTime(), date);
return new Date(date);
} else {
// unknown case, recheck in 24 hours
return DateUtils.addHours(now, 24);
}
}
public static Long averageTimeBetweenEntries(List<FeedEntry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
List<Long> timestamps = getSortedTimestamps(entries);
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < timestamps.size() - 1; i++) {
long diff = Math.abs(timestamps.get(i) - timestamps.get(i + 1));
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static List<Long> getSortedTimestamps(List<FeedEntry> entries) {
return entries.stream().map(t -> t.getUpdated().getTime()).sorted(Collections.reverseOrder()).collect(Collectors.toList());
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param url
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String url, String feedLink, String feedUrl) {
url = StringUtils.trimToNull(StringUtils.normalizeSpace(url));
if (url == null || url.startsWith("http")) {
return url;
}
String baseUrl = (feedLink == null || isRelative(feedLink)) ? feedUrl : feedLink;
if (baseUrl == null) {
return url;
}
String result = null;
try {
result = new URL(new URL(baseUrl), url).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : " + e.getMessage(), e);
result = url;
}
return result;
}
public static boolean isRelative(final String url) {
// the regex means "start with 'scheme://'"
return url.startsWith("/") || url.startsWith("#") || !url.matches("^\\w+\\:\\/\\/.*");
}
public static String getFaviconUrl(FeedSubscription subscription, String publicUrl) {
return removeTrailingSlash(publicUrl) + "/rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content, String publicUrl) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (href != null) {
String proxy = proxyImage(href, publicUrl);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url, String publicUrl) {
if (StringUtils.isBlank(url)) {
return url;
}
return removeTrailingSlash(publicUrl) + "/rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}

23
commafeed-server/src/main/java/com/commafeed/backend/feed/FetchedFeed.java Normal file
View File

@@ -0,0 +1,23 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FetchedFeed {
private Feed feed = new Feed();
private List<FeedEntry> entries = new ArrayList<>();
private String title;
private String urlAfterRedirect;
private long fetchDuration;
}

269
commafeed-server/src/main/java/com/commafeed/backend/feed/HtmlEntities.java Normal file
View File

@@ -0,0 +1,269 @@
package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
public class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[map.size()]);
NUMERIC_ENTITIES = map.values().toArray(new String[map.size()]);
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java Normal file
View File

@@ -0,0 +1,33 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import javax.persistence.TableGenerator;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}

129
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java Normal file
View File

@@ -0,0 +1,129 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDS")
@SuppressWarnings("serial")
@Getter
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Column(length = 2048, nullable = false)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Column(length = 2048)
private String link;
/**
* Last time we tried to fetch the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastEntryDate;
/**
* error message while retrieving the feed
*/
@Column(length = 1024)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Temporal(TemporalType.TIMESTAMP)
private Date disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
/**
* detected hub for pubsubhubbub
*/
@Column(length = 2048)
private String pushHub;
/**
* detected topic for pubsubhubbub
*/
@Column(length = 2048)
private String pushTopic;
@Column(name = "push_topic_hash", length = 2048)
private String pushTopicHash;
/**
* last time we subscribed for that topic on that hub
*/
@Temporal(TemporalType.TIMESTAMP)
private Date pushLastPing;
@OneToMany(mappedBy = "feed")
private Set<FeedSubscription> subscriptions;
}

43
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java Normal file
View File

@@ -0,0 +1,43 @@
package com.commafeed.backend.model;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDCATEGORIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@OneToMany(mappedBy = "parent")
private Set<FeedCategory> children;
@OneToMany(mappedBy = "category")
private Set<FeedSubscription> subscriptions;
private boolean collapsed;
private Integer position;
}

56
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java Normal file
View File

@@ -0,0 +1,56 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.util.Set;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@OneToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
@Temporal(TemporalType.TIMESTAMP)
private Date inserted;
@Temporal(TemporalType.TIMESTAMP)
private Date updated;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}

82
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java Normal file
View File

@@ -0,0 +1,82 @@
package com.commafeed.backend.model;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Lob;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.Type;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryContent extends AbstractModel {
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@Type(type = "org.hibernate.type.TextType")
private String content;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@Type(type = "org.hibernate.type.TextType")
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@OneToMany(mappedBy = "content")
private Set<FeedEntry> entries;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.append(categories, c.categories)
.build();
}
}

71
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java Normal file
View File

@@ -0,0 +1,71 @@
package com.commafeed.backend.model;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Transient
private boolean markable;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
/**
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Temporal(TemporalType.TIMESTAMP)
private Date entryInserted;
@Temporal(TemporalType.TIMESTAMP)
private Date entryUpdated;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
setUser(user);
setSubscription(subscription);
setEntry(entry);
setEntryInserted(entry.getInserted());
setEntryUpdated(entry.getUpdated());
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java Normal file
View File

@@ -0,0 +1,40 @@
package com.commafeed.backend.model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYTAGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}

46
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java Normal file
View File

@@ -0,0 +1,46 @@
package com.commafeed.backend.model;
import java.util.Set;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDSUBSCRIPTIONS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private Integer position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
}

29
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java Normal file
View File

@@ -0,0 +1,29 @@
package com.commafeed.backend.model;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
public class Models {
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy) {
LazyInitializer lazyInitializer = ((HibernateProxy) model).getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}

65
commafeed-server/src/main/java/com/commafeed/backend/model/User.java Normal file
View File

@@ -0,0 +1,65 @@
package com.commafeed.backend.model;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.apache.commons.lang3.time.DateUtils;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERS")
@SuppressWarnings("serial")
@Getter
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Column(length = 256, nullable = false)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Column(length = 8, nullable = false)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Temporal(TemporalType.TIMESTAMP)
private Date lastLogin;
@Temporal(TemporalType.TIMESTAMP)
private Date created;
@Column(length = 40)
private String recoverPasswordToken;
@Temporal(TemporalType.TIMESTAMP)
private Date recoverPasswordTokenDate;
@Column(name = "last_full_refresh")
@Temporal(TemporalType.TIMESTAMP)
private Date lastFullRefresh;
public boolean shouldRefreshFeedsAt(Date when) {
return lastFullRefresh == null || lastFullRefreshMoreThan30MinutesBefore(when);
}
private boolean lastFullRefreshMoreThan30MinutesBefore(Date when) {
return lastFullRefresh.before(DateUtils.addMinutes(when, -30));
}
}

43
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java Normal file
View File

@@ -0,0 +1,43 @@
package com.commafeed.backend.model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERROLES")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}

79
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java Normal file
View File

@@ -0,0 +1,79 @@
package com.commafeed.backend.model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.Lob;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import org.hibernate.annotations.Type;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
all, unread
}
public enum ReadingOrder {
asc, desc
}
public enum ViewMode {
title, expanded
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ViewMode viewMode;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
@Column(length = 32)
private String theme;
@Lob
@Column(length = Integer.MAX_VALUE)
@Type(type = "org.hibernate.type.TextType")
private String customCss;
@Column(name = "scroll_speed")
private int scrollSpeed;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean pocket;
private boolean instapaper;
private boolean buffer;
}

89
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java Normal file
View File

@@ -0,0 +1,89 @@
package com.commafeed.backend.opml;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OPMLExporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.1");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
Collections.sort(categories,
(e1, e2) -> ObjectUtils.firstNonNull(e1.getPosition(), 0) - ObjectUtils.firstNonNull(e2.getPosition(), 0));
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
Collections.sort(subscriptions,
(e1, e2) -> ObjectUtils.firstNonNull(e1.getPosition(), 0) - ObjectUtils.firstNonNull(e2.getPosition(), 0));
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).collect(Collectors.toList())) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).collect(Collectors.toList())) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.collect(Collectors.toList())) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.collect(Collectors.toList())) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}

93
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java Normal file
View File

@@ -0,0 +1,93 @@
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.commafeed.backend.service.FeedSubscriptionService.FeedSubscriptionException;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.WireFeedInput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final CacheService cache;
public void importOpml(User user, String xml) {
xml = xml.substring(xml.indexOf('<'));
WireFeedInput input = new WireFeedInput();
try {
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (FeedSubscriptionException e) {
throw e;
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
cache.invalidateUserRootCategory(user);
}
}

23
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Generator.java Normal file
View File

@@ -0,0 +1,23 @@
package com.commafeed.backend.rome;
import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
/**
* Add missing title to the generated OPML
*
*/
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {
super("opml_1.1");
}
@Override
protected Element generateHead(Opml opml) {
Element head = new Element("head");
addNotNullSimpleElement(head, "title", opml.getTitle());
return head;
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
/**
* Support for OPML 1.1 parsing
*
*/
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
if (e.getName().equals("opml")) {
return true;
}
return false;
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}

27
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionConverter.java Normal file
View File

@@ -0,0 +1,27 @@
package com.commafeed.backend.rome;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
/**
* Support description tag for RSS09
*
*/
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override
protected SyndEntry createSyndEntry(Item item, boolean preserveWireItem) {
SyndEntry entry = super.createSyndEntry(item, preserveWireItem);
Description desc = item.getDescription();
if (desc != null) {
SyndContentImpl syndDesc = new SyndContentImpl();
syndDesc.setValue(desc.getValue());
entry.setDescription(syndDesc);
}
return entry;
}
}

29
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionParser.java Normal file
View File

@@ -0,0 +1,29 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Element;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
/**
* Support description tag for RSS09
*
*/
public class RSS090DescriptionParser extends RSS090Parser {
@Override
protected Item parseItem(Element rssRoot, Element eItem, Locale locale) {
Item item = super.parseItem(rssRoot, eItem, locale);
Element e = eItem.getChild("description", getRSSNamespace());
if (e != null) {
Description desc = new Description();
desc.setValue(e.getText());
item.setDescription(desc);
}
return item;
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/rome/RSSRDF10Parser.java Normal file
View File

@@ -0,0 +1,49 @@
package com.commafeed.backend.rome;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";
private static final Namespace RSS_NS = Namespace.getNamespace(RSS_URI);
public RSSRDF10Parser() {
super("rss_1.0", RSS_NS);
}
@Override
public boolean isMyType(Document document) {
boolean ok = false;
Element rssRoot = document.getRootElement();
Namespace defaultNS = rssRoot.getNamespace();
List<Namespace> additionalNSs = Lists.newArrayList(rssRoot.getAdditionalNamespaces());
List<Element> children = rssRoot.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
Element child = children.get(0);
additionalNSs.add(child.getNamespace());
additionalNSs.addAll(child.getAdditionalNamespaces());
}
ok = defaultNS != null && defaultNS.equals(getRDFNamespace());
if (ok) {
if (additionalNSs == null) {
ok = false;
} else {
ok = false;
for (int i = 0; !ok && i < additionalNSs.size(); i++) {
ok = getRSSNamespace().equals(additionalNSs.get(i));
}
}
}
return ok;
}
}

113
commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java Normal file
View File

@@ -0,0 +1,113 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class DatabaseCleaningService {
private static final int BATCH_SIZE = 100;
private final SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
public long cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted = 0;
long entriesTotal = 0;
do {
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
int entriesDeleted = 0;
do {
entriesDeleted = UnitOfWork.call(sessionFactory, () -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
entriesTotal += entriesDeleted;
log.info("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = UnitOfWork.call(sessionFactory, () -> feedDAO.delete(feeds));
total += deleted;
log.info("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
return total;
}
public long cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
int deleted = 0;
do {
deleted = UnitOfWork.call(sessionFactory, () -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
total += deleted;
log.info("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
return total;
}
public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0;
while (true) {
List<FeedCapacity> feeds = UnitOfWork.call(sessionFactory,
() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = UnitOfWork.call(sessionFactory,
() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem)));
total += deleted;
remaining -= deleted;
log.info("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
return total;
}
public long cleanStatusesOlderThan(final Date olderThan) {
log.info("cleaning old read statuses");
long total = 0;
int deleted = 0;
do {
deleted = UnitOfWork.call(sessionFactory,
() -> feedEntryStatusDAO.delete(feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE)));
total += deleted;
log.info("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
return total;
}
}

48
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java Normal file
View File

@@ -0,0 +1,48 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(FeedEntryContent content, String baseUrl) {
content.setAuthor(FeedUtils.truncate(FeedUtils.handleContent(content.getAuthor(), baseUrl, true), 128));
content.setTitle(FeedUtils.truncate(FeedUtils.handleContent(content.getTitle(), baseUrl, true), 2048));
content.setContent(FeedUtils.handleContent(content.getContent(), baseUrl, false));
content.setMediaDescription(FeedUtils.handleContent(content.getMediaDescription(), baseUrl, false));
String contentHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent()));
content.setContentHash(contentHash);
String titleHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getTitle()));
content.setTitleHash(titleHash);
List<FeedEntryContent> existing = feedEntryContentDAO.findExisting(contentHash, titleHash);
Optional<FeedEntryContent> equivalentContent = existing.stream().filter(c -> content.equivalentTo(c)).findFirst();
if (equivalentContent.isPresent()) {
return equivalentContent.get();
}
feedEntryContentDAO.saveOrUpdate(content);
return content;
}
}

122
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java Normal file
View File

@@ -0,0 +1,122 @@
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script = null;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result = null;
try {
result = future.get(500, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

95
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java Normal file
View File

@@ -0,0 +1,95 @@
package com.commafeed.backend.service;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final CacheService cache;
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
cache.invalidateUnreadCount(sub);
cache.invalidateUserRootCategory(user);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Date olderThan, List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, false, null);
markList(statuses, olderThan);
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(user);
}
public void markStarredEntries(User user, Date olderThan) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan);
}
private void markList(List<FeedEntryStatus> statuses, Date olderThan) {
List<FeedEntryStatus> list = new ArrayList<>();
for (FeedEntryStatus status : statuses) {
if (!status.isRead()) {
Date entryDate = status.getEntry().getUpdated();
if (olderThan == null || entryDate == null || olderThan.after(entryDate)) {
status.setRead(true);
list.add(status);
}
}
}
feedEntryStatusDAO.saveOrUpdate(list);
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.collect(Collectors.toList());
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).collect(Collectors.toList());
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}

68
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java Normal file
View File

@@ -0,0 +1,68 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.Date;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.AbstractFaviconFetcher.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final Set<AbstractFaviconFetcher> faviconFetchers;
private Favicon defaultFavicon;
@Inject
public FeedService(FeedDAO feedDAO, Set<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(IOUtils.toByteArray(getClass().getResource("/images/default_favicon.gif")), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalized = FeedUtils.normalizeURL(url);
Feed feed = feedDAO.findByUrl(normalized);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setDisabledUntil(new Date(0));
feedDAO.saveOrUpdate(feed);
}
return feed;
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

124
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java Normal file
View File

@@ -0,0 +1,124 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedQueues queues;
private final CacheService cache;
private final CommaFeedConfiguration config;
public Feed subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public Feed subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public Feed subscribe(User user, String url, String title, FeedCategory category, int position) {
final String pubUrl = config.getApplicationSettings().getPublicUrl();
if (StringUtils.isBlank(pubUrl)) {
throw new FeedSubscriptionException("Public URL of this CommaFeed instance is not set");
}
if (url.startsWith(pubUrl)) {
throw new FeedSubscriptionException("Could not subscribe to a feed from this CommaFeed instance");
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
queues.add(feed, false);
cache.invalidateUserRootCategory(user);
return feed;
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
cache.invalidateUserRootCategory(user);
return true;
} else {
return false;
}
}
public void refreshAll(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
queues.add(feed, true);
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user).stream().collect(Collectors.toMap(FeedSubscription::getId, s -> getUnreadCount(user, s)));
}
private UnreadCount getUnreadCount(User user, FeedSubscription sub) {
UnreadCount count = cache.getUnreadCount(sub);
if (count == null) {
log.debug("unread count cache miss for {}", Models.getId(sub));
count = feedEntryStatusDAO.getUnreadCount(user, sub);
cache.setUnreadCount(sub, count);
}
return count;
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
}

67
commafeed-server/src/main/java/com/commafeed/backend/service/FeedUpdateService.java Normal file
View File

@@ -0,0 +1,67 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedUpdateService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
if (existing != null) {
return false;
}
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink());
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
return true;
}
}

64
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java Normal file
View File

@@ -0,0 +1,64 @@
package com.commafeed.backend.service;
import java.util.Optional;
import java.util.Properties;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
/**
* Mailing service
*
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class MailService {
private final CommaFeedConfiguration config;
public void sendMail(User user, String subject, String content) throws Exception {
ApplicationSettings settings = config.getApplicationSettings();
final String username = settings.getSmtpUserName();
final String password = settings.getSmtpPassword();
final String fromAddress = Optional.ofNullable(settings.getSmtpFromAddress()).orElse(settings.getSmtpUserName());
String dest = user.getEmail();
Properties props = new Properties();
props.put("mail.smtp.auth", "true");
props.put("mail.smtp.starttls.enable", "" + settings.isSmtpTls());
props.put("mail.smtp.host", settings.getSmtpHost());
props.put("mail.smtp.port", "" + settings.getSmtpPort());
Session session = Session.getInstance(props, new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
Message message = new MimeMessage(session);
message.setFrom(new InternetAddress(fromAddress, "CommaFeed"));
message.setRecipients(Message.RecipientType.TO, InternetAddress.parse(dest));
message.setSubject("CommaFeed - " + subject);
message.setContent(content, "text/html; charset=utf-8");
Transport.send(message);
}
}

95
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java Normal file
View File

@@ -0,0 +1,95 @@
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

91
commafeed-server/src/main/java/com/commafeed/backend/service/PubSubService.java Normal file
View File

@@ -0,0 +1,91 @@
package com.commafeed.backend.service;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.core.MediaType;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.frontend.resource.PubSubHubbubCallbackREST;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Sends push subscription requests. Callback is handled by {@link PubSubHubbubCallbackREST}
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class PubSubService {
private final CommaFeedConfiguration config;
private final FeedQueues queues;
public void subscribe(Feed feed) {
String hub = feed.getPushHub();
String topic = feed.getPushTopic();
String publicUrl = FeedUtils.removeTrailingSlash(config.getApplicationSettings().getPublicUrl());
log.debug("sending new pubsub subscription to {} for {}", hub, topic);
HttpPost post = new HttpPost(hub);
List<NameValuePair> nvp = new ArrayList<>();
nvp.add(new BasicNameValuePair("hub.callback", publicUrl + "/rest/push/callback"));
nvp.add(new BasicNameValuePair("hub.topic", topic));
nvp.add(new BasicNameValuePair("hub.mode", "subscribe"));
nvp.add(new BasicNameValuePair("hub.verify", "async"));
nvp.add(new BasicNameValuePair("hub.secret", ""));
nvp.add(new BasicNameValuePair("hub.verify_token", ""));
nvp.add(new BasicNameValuePair("hub.lease_seconds", ""));
post.setHeader(HttpHeaders.USER_AGENT, "CommaFeed");
post.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_FORM_URLENCODED);
CloseableHttpClient client = HttpGetter.newClient(20000);
CloseableHttpResponse response = null;
try {
post.setEntity(new UrlEncodedFormEntity(nvp));
response = client.execute(post);
int code = response.getStatusLine().getStatusCode();
if (code != 204 && code != 202 && code != 200) {
String message = EntityUtils.toString(response.getEntity());
String pushpressError = " is value is not allowed. You may only subscribe to";
if (code == 400 && StringUtils.contains(message, pushpressError)) {
String[] tokens = message.split(" ");
feed.setPushTopic(tokens[tokens.length - 1]);
queues.giveBack(feed);
log.debug("handled pushpress subfeed {} : {}", topic, feed.getPushTopic());
} else {
throw new Exception(
"Unexpected response code: " + code + " " + response.getStatusLine().getReasonPhrase() + " - " + message);
}
}
log.debug("subscribed to {} for {}", hub, topic);
} catch (Exception e) {
log.error("Could not subscribe to {} for {} : " + e.getMessage(), hub, topic);
} finally {
IOUtils.closeQuietly(response);
IOUtils.closeQuietly(client);
}
}
}

89
commafeed-server/src/main/java/com/commafeed/backend/service/StartupService.java Normal file
View File

@@ -0,0 +1,89 @@
package com.commafeed.backend.service;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import io.dropwizard.lifecycle.Managed;
import liquibase.Liquibase;
import liquibase.database.Database;
import liquibase.database.DatabaseFactory;
import liquibase.database.core.PostgresDatabase;
import liquibase.database.jvm.JdbcConnection;
import liquibase.resource.ClassLoaderResourceAccessor;
import liquibase.resource.ResourceAccessor;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class StartupService implements Managed {
private final SessionFactory sessionFactory;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
@Override
public void start() throws Exception {
updateSchema();
long count = UnitOfWork.call(sessionFactory, () -> userDAO.count());
if (count == 0) {
UnitOfWork.run(sessionFactory, this::initialData);
}
}
private void updateSchema() {
Session session = sessionFactory.openSession();
session.doWork(connection -> {
try {
JdbcConnection jdbcConnection = new JdbcConnection(connection);
Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(jdbcConnection);
if (database instanceof PostgresDatabase) {
database = new PostgresDatabase() {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
};
database.setConnection(jdbcConnection);
}
ResourceAccessor accessor = new ClassLoaderResourceAccessor(Thread.currentThread().getContextClassLoader());
try (Liquibase liq = new Liquibase("migrations.xml", accessor, database)) {
liq.update("prod");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
});
session.close();
}
private void initialData() {
log.info("Populating database with default values");
try {
userService.createAdminUser();
if (config.getApplicationSettings().getCreateDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
@Override
public void stop() throws Exception {
}
}

145
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java Normal file
View File

@@ -0,0 +1,145 @@
package com.commafeed.backend.service;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.getApplicationSettings().getAllowRegistrations(),
"Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(new Date());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Arrays.asList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return DigestUtils.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

46
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java Normal file
View File

@@ -0,0 +1,46 @@
package com.commafeed.backend.service.internal;
import java.util.Date;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.time.DateUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final CommaFeedConfiguration config;
public void executeFor(User user) {
Date lastLogin = user.getLastLogin();
Date now = new Date();
boolean saveUser = false;
// only update lastLogin field every hour in order to not
// invalidate the cache every time someone logs in
if (lastLogin == null || lastLogin.before(DateUtils.addHours(now, -1))) {
user.setLastLogin(now);
saveUser = true;
}
if (config.getApplicationSettings().getHeavyLoad() && user.shouldRefreshFeedsAt(now)) {
feedSubscriptionService.refreshAll(user);
user.setLastFullRefresh(now);
saveUser = true;
}
if (saveUser) {
userDAO.saveOrUpdate(user);
}
}
}

64
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java Normal file
View File

@@ -0,0 +1,64 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final SessionFactory sessionFactory;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.getApplicationSettings().getCreateDemoAccount()) {
return;
}
log.info("recreating demo user account");
UnitOfWork.run(sessionFactory, () -> {
User demoUser = userDAO.findByName(CommaFeedApplication.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

43
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java Normal file
View File

@@ -0,0 +1,43 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.task;
import java.util.Date;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Date threshold = config.getApplicationSettings().getUnreadThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java Normal file
View File

@@ -0,0 +1,38 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java Normal file
View File

@@ -0,0 +1,38 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java Normal file
View File

@@ -0,0 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = new Runnable() {
@Override
public void run() {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
};
log.info("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), getTimeUnit(),
getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}

10
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java Normal file
View File

@@ -0,0 +1,10 @@
package com.commafeed.backend.urlprovider;
/**
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider {
String get(String url, String urlContent);
}

28
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java Normal file
View File

@@ -0,0 +1,28 @@
package com.commafeed.backend.urlprovider;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public String get(String url, String urlContent) {
String foundUrl = null;
Document doc = Jsoup.parse(urlContent, url);
String root = doc.children().get(0).tagName();
if ("html".equals(root)) {
Elements atom = doc.select("link[type=application/atom+xml]");
Elements rss = doc.select("link[type=application/rss+xml]");
if (!atom.isEmpty()) {
foundUrl = atom.get(0).attr("abs:href");
} else if (!rss.isEmpty()) {
foundUrl = rss.get(0).attr("abs:href");
}
}
return foundUrl;
}
}

22
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.backend.urlprovider;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final Pattern REGEXP = Pattern.compile("(.*\\byoutube\\.com)\\/channel\\/([^\\/]+)", Pattern.CASE_INSENSITIVE);
@Override
public String get(String url, String urlContent) {
Matcher matcher = REGEXP.matcher(url);
return matcher.find() ? matcher.group(1) + "/feeds/videos.xml?channel_id=" + matcher.group(2) : null;
}
}

59
commafeed-server/src/main/java/com/commafeed/frontend/auth/PasswordConstraintValidator.java Normal file
View File

@@ -0,0 +1,59 @@
package com.commafeed.frontend.auth;
import java.util.List;
import javax.validation.ConstraintValidator;
import javax.validation.ConstraintValidatorContext;
import org.apache.commons.lang3.StringUtils;
import org.passay.CharacterRule;
import org.passay.EnglishCharacterData;
import org.passay.LengthRule;
import org.passay.PasswordData;
import org.passay.PasswordValidator;
import org.passay.RuleResult;
import org.passay.WhitespaceRule;
public class PasswordConstraintValidator implements ConstraintValidator<ValidPassword, String> {
@Override
public void initialize(ValidPassword constraintAnnotation) {
// nothing to do
}
@Override
public boolean isValid(String value, ConstraintValidatorContext context) {
if (StringUtils.isBlank(value)) {
return true;
}
PasswordValidator validator = buildPasswordValidator();
RuleResult result = validator.validate(new PasswordData(value));
if (result.isValid()) {
return true;
}
List<String> messages = validator.getMessages(result);
String message = String.join(System.lineSeparator(), messages);
context.buildConstraintViolationWithTemplate(message).addConstraintViolation().disableDefaultConstraintViolation();
return false;
}
private PasswordValidator buildPasswordValidator() {
return new PasswordValidator(
// length
new LengthRule(8, 128),
// 1 uppercase char
new CharacterRule(EnglishCharacterData.UpperCase, 1),
// 1 lowercase char
new CharacterRule(EnglishCharacterData.LowerCase, 1),
// 1 digit
new CharacterRule(EnglishCharacterData.Digit, 1),
// 1 special char
new CharacterRule(EnglishCharacterData.Special, 1),
// no whitespace
new WhitespaceRule());
}
}

22
commafeed-server/src/main/java/com/commafeed/frontend/auth/SecurityCheck.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.frontend.auth;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.commafeed.backend.model.UserRole.Role;
@Inherited
@Target({ ElementType.PARAMETER })
@Retention(RetentionPolicy.RUNTIME)
public @interface SecurityCheck {
/**
* Roles needed.
*/
Role value() default Role.USER;
boolean apiKeyAllowed() default false;
}

99
commafeed-server/src/main/java/com/commafeed/frontend/auth/SecurityCheckFactory.java Normal file
View File

@@ -0,0 +1,99 @@
package com.commafeed.frontend.auth;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.glassfish.jersey.server.ContainerRequest;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.UserService;
import com.commafeed.frontend.session.SessionHelper;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class SecurityCheckFactory implements Function<ContainerRequest, User> {
private static final String PREFIX = "Basic";
private final UserService userService;
private final HttpServletRequest request;
private final Role role;
private final boolean apiKeyAllowed;
@Override
public User apply(ContainerRequest req) {
Optional<User> user = apiKeyLogin();
if (!user.isPresent()) {
user = basicAuthenticationLogin();
}
if (!user.isPresent()) {
user = cookieSessionLogin(new SessionHelper(request));
}
if (user.isPresent()) {
Set<Role> roles = userService.getRoles(user.get());
if (roles.contains(role)) {
return user.get();
} else {
throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN)
.entity("You don't have the required role to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
}
} else {
throw new WebApplicationException(Response.status(Response.Status.UNAUTHORIZED)
.entity("Credentials are required to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
}
}
Optional<User> cookieSessionLogin(SessionHelper sessionHelper) {
Optional<User> loggedInUser = sessionHelper.getLoggedInUser();
if (loggedInUser.isPresent()) {
userService.performPostLoginActivities(loggedInUser.get());
}
return loggedInUser;
}
private Optional<User> basicAuthenticationLogin() {
String header = request.getHeader(HttpHeaders.AUTHORIZATION);
if (header != null) {
int space = header.indexOf(' ');
if (space > 0) {
String method = header.substring(0, space);
if (PREFIX.equalsIgnoreCase(method)) {
byte[] decodedBytes = Base64.getDecoder().decode(header.substring(space + 1));
String decoded = new String(decodedBytes, StandardCharsets.ISO_8859_1);
int i = decoded.indexOf(':');
if (i > 0) {
String username = decoded.substring(0, i);
String password = decoded.substring(i + 1);
return userService.login(username, password);
}
}
}
}
return Optional.empty();
}
private Optional<User> apiKeyLogin() {
String apiKey = request.getParameter("apiKey");
if (apiKey != null && apiKeyAllowed) {
return userService.login(apiKey);
}
return Optional.empty();
}
}

63
commafeed-server/src/main/java/com/commafeed/frontend/auth/SecurityCheckFactoryProvider.java Normal file
View File

@@ -0,0 +1,63 @@
package com.commafeed.frontend.auth;
import java.util.function.Function;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ContainerRequest;
import org.glassfish.jersey.server.internal.inject.AbstractValueParamProvider;
import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractorProvider;
import org.glassfish.jersey.server.model.Parameter;
import org.glassfish.jersey.server.spi.internal.ValueParamProvider;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
@Singleton
public class SecurityCheckFactoryProvider extends AbstractValueParamProvider {
private UserService userService;
private HttpServletRequest request;
@Inject
public SecurityCheckFactoryProvider(final MultivaluedParameterExtractorProvider extractorProvider, UserService userService,
HttpServletRequest request) {
super(() -> extractorProvider, Parameter.Source.UNKNOWN);
this.userService = userService;
this.request = request;
}
@Override
protected Function<ContainerRequest, ?> createValueProvider(Parameter parameter) {
final Class<?> classType = parameter.getRawType();
SecurityCheck securityCheck = parameter.getAnnotation(SecurityCheck.class);
if (securityCheck == null) {
return null;
}
if (!classType.isAssignableFrom(User.class)) {
return null;
}
return new SecurityCheckFactory(userService, request, securityCheck.value(), securityCheck.apiKeyAllowed());
}
@RequiredArgsConstructor
public static class Binder extends AbstractBinder {
private final UserService userService;
@Override
protected void configure() {
bind(SecurityCheckFactoryProvider.class).to(ValueParamProvider.class).in(Singleton.class);
bind(userService).to(UserService.class);
}
}
}

23
commafeed-server/src/main/java/com/commafeed/frontend/auth/ValidPassword.java Normal file
View File

@@ -0,0 +1,23 @@
package com.commafeed.frontend.auth;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import javax.validation.Constraint;
import javax.validation.Payload;
@Documented
@Constraint(validatedBy = PasswordConstraintValidator.class)
@Target({ ElementType.TYPE, ElementType.FIELD, ElementType.ANNOTATION_TYPE })
@Retention(RetentionPolicy.RUNTIME)
public @interface ValidPassword {
String message() default "Invalid Password";
Class<?>[] groups() default {};
Class<? extends Payload>[] payload() default {};
}

36
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java Normal file
View File

@@ -0,0 +1,36 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Entry details")
@Data
public class Category implements Serializable {
@ApiModelProperty(value = "category id", required = true)
private String id;
@ApiModelProperty(value = "parent category id")
private String parentId;
@ApiModelProperty(value = "category id", required = true)
private String name;
@ApiModelProperty(value = "category children categories", required = true)
private List<Category> children = new ArrayList<>();
@ApiModelProperty(value = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>();
@ApiModelProperty(value = "wether the category is expanded or collapsed", required = true)
private boolean expanded;
@ApiModelProperty(value = "position of the category in the list", required = true)
private Integer position;
}

48
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java Normal file
View File

@@ -0,0 +1,48 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "List of entries with some metadata")
@Data
public class Entries implements Serializable {
@ApiModelProperty(value = "name of the feed or the category requested", required = true)
private String name;
@ApiModelProperty(value = "error or warning message")
private String message;
@ApiModelProperty(value = "times the server tried to refresh the feed and failed", required = true)
private int errorCount;
@ApiModelProperty(value = "URL of the website, extracted from the feed", required = true)
private String feedLink;
@ApiModelProperty(value = "list generation timestamp", required = true)
private long timestamp;
@ApiModelProperty(value = "if the query has more elements", required = true)
private boolean hasMore;
@ApiModelProperty(value = "the requested offset")
private int offset;
@ApiModelProperty(value = "the requested limit")
private int limit;
@ApiModelProperty(value = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>();
@ApiModelProperty(
value = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
}

175
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java Normal file
View File

@@ -0,0 +1,175 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Entry details")
@Data
public class Entry implements Serializable {
@ApiModelProperty(value = "entry id", required = true)
private String id;
@ApiModelProperty(value = "entry guid", required = true)
private String guid;
@ApiModelProperty(value = "entry title", required = true)
private String title;
@ApiModelProperty(value = "entry content", required = true)
private String content;
@ApiModelProperty(value = "comma-separated list of categories")
private String categories;
@ApiModelProperty(value = "wether entry content and title are rtl", required = true)
private boolean rtl;
@ApiModelProperty(value = "entry author")
private String author;
@ApiModelProperty(value = "entry enclosure url, if any")
private String enclosureUrl;
@ApiModelProperty(value = "entry enclosure mime type, if any")
private String enclosureType;
@ApiModelProperty(value = "entry media description, if any")
private String mediaDescription;
@ApiModelProperty(value = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@ApiModelProperty(value = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@ApiModelProperty(value = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@ApiModelProperty(value = "entry publication date", dataType = "number", required = true)
private Date date;
@ApiModelProperty(value = "entry insertion date in the database", dataType = "number", required = true)
private Date insertedDate;
@ApiModelProperty(value = "feed id", required = true)
private String feedId;
@ApiModelProperty(value = "feed name", required = true)
private String feedName;
@ApiModelProperty(value = "this entry's feed url", required = true)
private String feedUrl;
@ApiModelProperty(value = "this entry's website url", required = true)
private String feedLink;
@ApiModelProperty(value = "The favicon url to use for this feed", required = true)
private String iconUrl;
@ApiModelProperty(value = "entry url", required = true)
private String url;
@ApiModelProperty(value = "read status", required = true)
private boolean read;
@ApiModelProperty(value = "starred status", required = true)
private boolean starred;
@ApiModelProperty(value = "wether the entry is still markable (old entry statuses are discarded)", required = true)
private boolean markable;
@ApiModelProperty(value = "tags", required = true)
private List<String> tags;
public static Entry build(FeedEntryStatus status, String publicUrl, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getUpdated());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub, publicUrl));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).collect(Collectors.toList()));
if (content != null) {
entry.setRtl(FeedUtils.isRTL(feedEntry));
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent(), publicUrl) : content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && StringUtils.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl(), publicUrl)
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(
proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl(), publicUrl) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
public SyndEntry asRss() {
SyndEntry entry = new SyndEntryImpl();
entry.setUri(getGuid());
entry.setTitle(getTitle());
entry.setAuthor(getAuthor());
SyndContentImpl content = new SyndContentImpl();
content.setValue(getContent());
entry.setContents(Arrays.<SyndContent> asList(content));
if (getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(getEnclosureType());
enclosure.setUrl(getEnclosureUrl());
entry.setEnclosures(Arrays.<SyndEnclosure> asList(enclosure));
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate());
return entry;
}
}

20
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java Normal file
View File

@@ -0,0 +1,20 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Feed details")
@Data
public class FeedInfo implements Serializable {
@ApiModelProperty(value = "url", required = true)
private String url;
@ApiModelProperty(value = "title", required = true)
private String title;
}

32
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java Normal file
View File

@@ -0,0 +1,32 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Server infos")
@Data
public class ServerInfo implements Serializable {
@ApiModelProperty
private String announcement;
@ApiModelProperty(required = true)
private String version;
@ApiModelProperty(required = true)
private String gitCommit;
@ApiModelProperty(required = true)
private boolean allowRegistrations;
@ApiModelProperty
private String googleAnalyticsCode;
@ApiModelProperty(required = true)
private boolean smtpEnabled;
}

64
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java Normal file
View File

@@ -0,0 +1,64 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "User settings")
@Data
public class Settings implements Serializable {
@ApiModelProperty(value = "user's preferred language, english if none", required = true)
private String language;
@ApiModelProperty(value = "user reads all entries or unread entries only", allowableValues = "all,unread", required = true)
private String readingMode;
@ApiModelProperty(value = "user reads entries in ascending or descending order", allowableValues = "asc,desc", required = true)
private String readingOrder;
@ApiModelProperty(value = "user viewing mode, either title-only or expande view", allowableValues = "title,expanded", required = true)
private String viewMode;
@ApiModelProperty(value = "user wants category and feeds with no unread entries shown", required = true)
private boolean showRead;
@ApiModelProperty(value = "In expanded view, scroll through entries mark them as read", required = true)
private boolean scrollMarks;
@ApiModelProperty(value = "user's selected theme")
private String theme;
@ApiModelProperty(value = "user's custom css for the website")
private String customCss;
@ApiModelProperty(value = "user's preferred scroll speed when navigating between entries", required = true)
private int scrollSpeed;
@ApiModelProperty(required = true)
private boolean email;
@ApiModelProperty(required = true)
private boolean gmail;
@ApiModelProperty(required = true)
private boolean facebook;
@ApiModelProperty(required = true)
private boolean twitter;
@ApiModelProperty(required = true)
private boolean tumblr;
@ApiModelProperty(required = true)
private boolean pocket;
@ApiModelProperty(required = true)
private boolean instapaper;
@ApiModelProperty(required = true)
private boolean buffer;
}

87
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java Normal file
View File

@@ -0,0 +1,87 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "User information")
@Data
public class Subscription implements Serializable {
@ApiModelProperty(value = "subscription id", required = true)
private Long id;
@ApiModelProperty(value = "subscription name", required = true)
private String name;
@ApiModelProperty(value = "error message while fetching the feed", required = true)
private String message;
@ApiModelProperty(value = "error count", required = true)
private int errorCount;
@ApiModelProperty(value = "last time the feed was refreshed", dataType = "number", required = true)
private Date lastRefresh;
@ApiModelProperty(
value = "next time the feed refresh is planned, null if refresh is already queued",
dataType = "number",
required = true)
private Date nextRefresh;
@ApiModelProperty(value = "this subscription's feed url", required = true)
private String feedUrl;
@ApiModelProperty(value = "this subscription's website url", required = true)
private String feedLink;
@ApiModelProperty(value = "The favicon url to use for this feed", required = true)
private String iconUrl;
@ApiModelProperty(value = "unread count", required = true)
private long unread;
@ApiModelProperty(value = "category id")
private String categoryId;
@ApiModelProperty("position of the subscription's in the list")
private Integer position;
@ApiModelProperty(value = "date of the newest item", dataType = "number")
private Date newestItemTime;
@ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, String publicUrl, UnreadCount unreadCount) {
Date now = new Date();
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription, publicUrl));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh((feed.getDisabledUntil() != null && feed.getDisabledUntil().before(now)) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
}

33
commafeed-server/src/main/java/com/commafeed/frontend/model/UnreadCount.java Normal file
View File

@@ -0,0 +1,33 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Unread count")
@Data
public class UnreadCount implements Serializable {
@ApiModelProperty
private long feedId;
@ApiModelProperty
private long unreadCount;
@ApiModelProperty(dataType = "number")
private Date newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Date newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}

Some files were not shown because too many files have changed in this diff Show More