Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-10-07 12:53:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

use google java formatter to avoid downloading eclipse during build

Browse Source
This commit is contained in:
Athou
2026-07-05 10:18:30 +02:00
parent f4dc279ec9
commit 1bd9b71215
200 changed files with 13923 additions and 12754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

401
commafeed-server/dev/EclipseCodeFormatter.xml
View File

@@ -1,401 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<profiles version="23">
<profile kind="CodeFormatterProfile" name="CommaFeed" version="23">
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.align_with_spaces" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
<setting id="org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_record_components" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_logical_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line" value="one_line_if_empty"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_shift_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_parameters" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_loops" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_switch_case_arrow_operator" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_enum_constant" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.text_block_indentation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_module_statements" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_permitted_types" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_annotations" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_assertion_message_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines" value="2147483647"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_not_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_arguments" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_permitted_types_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_record_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.javadoc_do_not_separate_block_tags" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_tag_description" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_record_constructor" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_string_concatenation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_record_components" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_additive_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_record_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.keep_switch_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_switch_case_with_arrow" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_method_body_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_additive_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_constructor" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_record_declaration_on_one_line" value="one_line_if_empty"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_lambda_body" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_relational_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.align_selector_in_method_invocation_on_expression_first_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_record_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_switch_case_with_arrow_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_switch_case_with_colon" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_conditional_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_shift_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.keep_code_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_record_components" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_assignment_operator" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_switch_case_with_arrow" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_record_constructor_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assertion_message" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_record_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_relational_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_logical_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_permitted_types" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="tab"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_string_concatenation" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
</profile>
</profiles>

23
commafeed-server/dev/checkstyle.xml
View File

@@ -1,8 +1,8 @@
<?xml version="1.0"?> <?xml version="1.0"?>
<!DOCTYPE module PUBLIC <!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd"> "https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker"> <module name="Checker">
<property name="charset" value="UTF-8" /> <property name="charset" value="UTF-8" />
<property name="fileExtensions" value="java" /> <property name="fileExtensions" value="java" />
@@ -40,11 +40,6 @@
<module name="AvoidStarImport" /> <module name="AvoidStarImport" />
<module name="AvoidStaticImport" /> <module name="AvoidStaticImport" />
<module name="IllegalImport" /> <module name="IllegalImport" />
<module name="ImportOrder">
<property name="groups" value="/^java\./,javax,jakarta,org,com" />
<property name="ordered" value="true" />
<property name="separated" value="true" />
</module>
<module name="RedundantImport" /> <module name="RedundantImport" />
<module name="UnusedImports" /> <module name="UnusedImports" />
@@ -60,7 +55,7 @@
<module name="EmptyCatchBlock"> <module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="ignore|ignored" /> <property name="exceptionVariableName" value="ignore|ignored" />
<message key="catch.block.empty" <message key="catch.block.empty"
value="Empty catch block. You can use the name 'ignore' or 'ignored' for the exception variable if you really want an empty catch block, but you should strongly consider at the very least logging something." /> value="Empty catch block. You can use the name 'ignore' or 'ignored' for the exception variable if you really want an empty catch block, but you should strongly consider at the very least logging something." />
</module> </module>
<module name="LeftCurly" /> <module name="LeftCurly" />
<module name="NeedBraces" /> <module name="NeedBraces" />
@@ -76,7 +71,7 @@
<module name="FallThrough" /> <module name="FallThrough" />
<module name="IllegalInstantiation"> <module name="IllegalInstantiation">
<property name="classes" <property name="classes"
value="java.lang.Boolean, java.lang.Byte, java.lang.Character, java.lang.Double, java.lang.Float, java.lang.Integer, java.lang.Long, java.lang.Short" /> value="java.lang.Boolean, java.lang.Byte, java.lang.Character, java.lang.Double, java.lang.Float, java.lang.Integer, java.lang.Long, java.lang.Short" />
</module> </module>
<module name="IllegalType" /> <module name="IllegalType" />
<module name="ModifiedControlVariable"> <module name="ModifiedControlVariable">
@@ -113,14 +108,6 @@
<module name="NoLineWrap" /> <module name="NoLineWrap" />
<module name="NoWhitespaceBefore" /> <module name="NoWhitespaceBefore" />
<module name="ParenPad" /> <module name="ParenPad" />
<module name="RegexpSinglelineJava">
<property name="format" value="^\t* +\t*\S" />
<property name="message" value="Line has leading space characters; indentation should be performed with tabs only." />
<property name="ignoreComments" value="true" />
</module>
<module name="WhitespaceAround">
<property name="allowEmptyTypes" value="true" />
</module>
</module> </module>
</module> </module>

7
commafeed-server/dev/eclipse.importorder
View File

@@ -1,7 +0,0 @@
#Organize Import Order
#Wed Jan 29 15:15:04 CET 2025
0=java
1=javax
2=jakarta
3=org
4=com

9
commafeed-server/pom.xml
View File

@@ -285,12 +285,9 @@
<configuration> <configuration>
<encoding>UTF-8</encoding> <encoding>UTF-8</encoding>
<java> <java>
<eclipse> <googleJavaFormat>
<file>${project.basedir}/dev/EclipseCodeFormatter.xml</file> <style>AOSP</style>
</eclipse> </googleJavaFormat>
<importOrder>
<file>${project.basedir}/dev/eclipse.importorder</file>
</importOrder>
</java> </java>
</configuration> </configuration>
</plugin> </plugin>

42
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
View File

@@ -1,15 +1,13 @@
package com.commafeed; package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine; import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.ImageProxyUrl; import com.commafeed.backend.feed.ImageProxyUrl;
import com.commafeed.backend.task.TaskScheduler; import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator; import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent; import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -18,28 +16,28 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor @RequiredArgsConstructor
public class CommaFeedApplication { public class CommaFeedApplication {
private final FeedRefreshEngine feedRefreshEngine; private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler; private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) { public void start(@Observes StartupEvent ev) {
log.info("starting up..."); log.info("starting up...");
PasswordConstraintValidator.setMinimumPasswordLength(config.users().minimumPasswordLength()); PasswordConstraintValidator.setMinimumPasswordLength(
config.users().minimumPasswordLength());
if (config.imageProxyEnabled()) { if (config.imageProxyEnabled()) {
ImageProxyUrl.generateKey(); ImageProxyUrl.generateKey();
} }
feedRefreshEngine.start(); feedRefreshEngine.start();
taskScheduler.start(); taskScheduler.start();
} }
public void stop(@Observes ShutdownEvent ev) { public void stop(@Observes ShutdownEvent ev) {
log.info("shutting down..."); log.info("shutting down...");
feedRefreshEngine.stop();
taskScheduler.stop();
}
feedRefreshEngine.stop();
taskScheduler.stop();
}
} }

622
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -1,406 +1,360 @@
package com.commafeed; package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator; import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection; import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase; import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot; import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize; import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping; import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault; import io.smallrye.config.WithDefault;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
/** /**
* CommaFeed configuration * CommaFeed configuration
* *
* Default values are for production, they can be overridden in application.properties for other profiles * <p>Default values are for production, they can be overridden in application.properties for other
* profiles
*/ */
@ConfigMapping(prefix = "commafeed") @ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME) @ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration { public interface CommaFeedConfiguration {
/** /**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers. * Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/ */
@WithDefault("true") @WithDefault("true")
boolean hideFromWebCrawlers(); boolean hideFromWebCrawlers();
/** /**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser. * If enabled, images in feed entries will be proxied through the server instead of accessed
* * directly by the browser.
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed. *
*/ * <p>This is useful if commafeed is accessed through a restricting proxy that blocks some feeds
@WithDefault("false") * that are followed.
boolean imageProxyEnabled(); */
@WithDefault("false")
boolean imageProxyEnabled();
/** /**
* Enable password recovery via email. * Enable password recovery via email.
* *
* Quarkus mailer will need to be configured. * <p>Quarkus mailer will need to be configured.
*/ */
@WithDefault("false") @WithDefault("false")
boolean passwordRecoveryEnabled(); boolean passwordRecoveryEnabled();
/** /** Message displayed in a notification at the bottom of the page. */
* Message displayed in a notification at the bottom of the page. Optional<String> announcement();
*/
Optional<String> announcement();
/** /** Google Auth key for fetching Youtube channel favicons. */
* Google Auth key for fetching Youtube channel favicons. Optional<String> googleAuthKey();
*/
Optional<String> googleAuthKey();
/** /** HTTP client configuration */
* HTTP client configuration @ConfigDocSection
*/ HttpClient httpClient();
@ConfigDocSection
HttpClient httpClient();
/** /** Feed refresh engine settings. */
* Feed refresh engine settings. @ConfigDocSection
*/ FeedRefresh feedRefresh();
@ConfigDocSection
FeedRefresh feedRefresh();
/** /** Push notification settings. */
* Push notification settings. @ConfigDocSection
*/ PushNotifications pushNotifications();
@ConfigDocSection
PushNotifications pushNotifications();
/** /** Database settings. */
* Database settings. @ConfigDocSection
*/ Database database();
@ConfigDocSection
Database database();
/** /** Users settings. */
* Users settings. @ConfigDocSection
*/ Users users();
@ConfigDocSection
Users users();
/** /** Websocket settings. */
* Websocket settings. @ConfigDocSection
*/ Websocket websocket();
@ConfigDocSection
Websocket websocket();
/** /**
* Duration to wait for the feed refresh engine and the task scheduler to stop when the application is shutting down. * Duration to wait for the feed refresh engine and the task scheduler to stop when the
*/ * application is shutting down.
@WithDefault("2s") */
Duration shutdownTimeout(); @WithDefault("2s")
Duration shutdownTimeout();
interface HttpClient { interface HttpClient {
/** /**
* User-Agent string that will be used by the http client, leave empty for the default one. * User-Agent string that will be used by the http client, leave empty for the default one.
*/ */
Optional<String> userAgent(); Optional<String> userAgent();
/** /** Time to wait for a connection to be established. */
* Time to wait for a connection to be established. @WithDefault("5s")
*/ Duration connectTimeout();
@WithDefault("5s")
Duration connectTimeout();
/** /** Time to wait for SSL handshake to complete. */
* Time to wait for SSL handshake to complete. @WithDefault("5s")
*/ Duration sslHandshakeTimeout();
@WithDefault("5s")
Duration sslHandshakeTimeout();
/** /** Time to wait between two packets before timeout. */
* Time to wait between two packets before timeout. @WithDefault("10s")
*/ Duration socketTimeout();
@WithDefault("10s")
Duration socketTimeout();
/** /** Time to wait for the full response to be received. */
* Time to wait for the full response to be received. @WithDefault("10s")
*/ Duration responseTimeout();
@WithDefault("10s")
Duration responseTimeout();
/** /** Time to live for a connection in the pool. */
* Time to live for a connection in the pool. @WithDefault("30s")
*/ Duration connectionTimeToLive();
@WithDefault("30s")
Duration connectionTimeToLive();
/** /** Time between eviction runs for idle connections. */
* Time between eviction runs for idle connections. @WithDefault("1m")
*/ Duration idleConnectionsEvictionInterval();
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/** /**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed. * If a feed is larger than this, it will be discarded to prevent memory issues while
*/ * parsing the feed.
@WithDefault("5M") */
MemorySize maxResponseSize(); @WithDefault("5M")
MemorySize maxResponseSize();
/** /**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal * Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks,
* resources. * which could potentially expose internal resources.
* *
* You may want to enable this if you host a public instance of CommaFeed with registrations open. * <p>You may want to enable this if you host a public instance of CommaFeed with
*/ * registrations open.
@WithDefault("false") */
boolean blockLocalAddresses(); @WithDefault("false")
boolean blockLocalAddresses();
/** /** HTTP client cache configuration */
* HTTP client cache configuration @ConfigDocSection
*/ HttpClientCache cache();
@ConfigDocSection }
HttpClientCache cache();
}
interface HttpClientCache { interface HttpClientCache {
/** /**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the * Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts
* first time or when clicking "fetch all my feeds now"). * (e.g. when subscribing to a feed for the first time or when clicking "fetch all my feeds
*/ * now").
@WithDefault("true") */
boolean enabled(); @WithDefault("true")
boolean enabled();
/** /** Maximum amount of memory the cache can use. */
* Maximum amount of memory the cache can use. @WithDefault("10M")
*/ MemorySize maximumMemorySize();
@WithDefault("10M")
MemorySize maximumMemorySize();
/** /** Duration after which an entry is removed from the cache. */
* Duration after which an entry is removed from the cache. @WithDefault("1m")
*/ Duration expiration();
@WithDefault("1m") }
Duration expiration();
}
interface FeedRefresh { interface FeedRefresh {
/** /** Default amount of time CommaFeed will wait before refreshing a feed. */
* Default amount of time CommaFeed will wait before refreshing a feed. @WithDefault("5m")
*/ Duration interval();
@WithDefault("5m")
Duration interval();
/** /**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when: * Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an
* * upper bound when:
* <ul> *
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li> * <ul>
* <li>we receive a Cache-Control header from the feed</li> * <li>an error occurs while refreshing a feed and we're backing off exponentially
* <li>we receive a Retry-After header from the feed</li> * <li>we receive a Cache-Control header from the feed
* </ul> * <li>we receive a Retry-After header from the feed
*/ * </ul>
@WithDefault("4h") */
Duration maxInterval(); @WithDefault("4h")
Duration maxInterval();
/** /**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since * If enabled, CommaFeed will calculate the next refresh time based on the feed's average
* the last entry was published. The interval will be sometimes between the default refresh interval * time between entries and the time since the last entry was published. The interval will
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`). * be sometimes between the default refresh interval (`commafeed.feed-refresh.interval`) and
* * the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
* See {@link FeedRefreshIntervalCalculator} for details. *
*/ * <p>See {@link FeedRefreshIntervalCalculator} for details.
@WithDefault("true") */
boolean intervalEmpirical(); @WithDefault("true")
boolean intervalEmpirical();
/** /** Feed refresh engine error handling settings. */
* Feed refresh engine error handling settings. @ConfigDocSection
*/ FeedRefreshErrorHandling errors();
@ConfigDocSection
FeedRefreshErrorHandling errors();
/** /** Amount of http threads used to fetch feeds. */
* Amount of http threads used to fetch feeds. @Min(1)
*/ @WithDefault("3")
@Min(1) int httpThreads();
@WithDefault("3")
int httpThreads();
/** /** Amount of threads used to insert new entries in the database. */
* Amount of threads used to insert new entries in the database. @Min(1)
*/ @WithDefault("1")
@Min(1) int databaseThreads();
@WithDefault("1")
int databaseThreads();
/** /**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again. * Duration after which a user is considered inactive. Feeds for inactive users are not
* * refreshed until they log in again.
* 0 to disable. *
*/ * <p>0 to disable.
@WithDefault("0") */
Duration userInactivityPeriod(); @WithDefault("0")
Duration userInactivityPeriod();
/** /**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out. * Duration after which the evaluation of a filtering expresion to mark an entry as read is
*/ * considered to have timed out.
@WithDefault("500ms") */
Duration filteringExpressionEvaluationTimeout(); @WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/** /**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds. * Duration after which the "Fetch all my feeds now" action is available again after use to
*/ * avoid spamming feeds.
@WithDefault("0") */
Duration forceRefreshCooldownDuration(); @WithDefault("0")
} Duration forceRefreshCooldownDuration();
}
interface PushNotifications { interface PushNotifications {
/** /** Whether to enable push notifications to notify users of new entries in their feeds. */
* Whether to enable push notifications to notify users of new entries in their feeds. @WithDefault("true")
*/ boolean enabled();
@WithDefault("true")
boolean enabled();
/** /** Amount of threads used to send external notifications about new entries. */
* Amount of threads used to send external notifications about new entries. @Min(1)
*/ @WithDefault("5")
@Min(1) int threads();
@WithDefault("5")
int threads();
/** /**
* Maximum amount of notifications that can be queued before new notifications are discarded. * Maximum amount of notifications that can be queued before new notifications are
*/ * discarded.
@Min(1) */
@WithDefault("100") @Min(1)
int queueCapacity(); @WithDefault("100")
} int queueCapacity();
}
interface FeedRefreshErrorHandling { interface FeedRefreshErrorHandling {
/** /** Number of retries before backoff is applied. */
* Number of retries before backoff is applied. @Min(0)
*/ @WithDefault("3")
@Min(0) int retriesBeforeBackoff();
@WithDefault("3")
int retriesBeforeBackoff();
/** /**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch. * Duration to wait before retrying after an error. Will be multiplied by the number of
*/ * errors since the last successful fetch.
@WithDefault("1h") */
Duration backoffInterval(); @WithDefault("1h")
} Duration backoffInterval();
}
interface Database { interface Database {
/** /**
* Timeout applied to all database queries. * Timeout applied to all database queries.
* *
* 0 to disable. * <p>0 to disable.
*/ */
@WithDefault("0") @WithDefault("0")
Duration queryTimeout(); Duration queryTimeout();
/** /** Database cleanup settings. */
* Database cleanup settings. @ConfigDocSection
*/ Cleanup cleanup();
@ConfigDocSection
Cleanup cleanup();
interface Cleanup { interface Cleanup {
/** /**
* Maximum age of feed entries in the database. Older entries will be deleted. * Maximum age of feed entries in the database. Older entries will be deleted.
* *
* 0 to disable. * <p>0 to disable.
*/ */
@WithDefault("365d") @WithDefault("365d")
Duration entriesMaxAge(); Duration entriesMaxAge();
/** /**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted. * Maximum age of feed entry statuses (read/unread) in the database. Older statuses will
* * be deleted.
* 0 to disable. *
*/ * <p>0 to disable.
@WithDefault("0") */
Duration statusesMaxAge(); @WithDefault("0")
Duration statusesMaxAge();
/** /**
* Maximum number of entries per feed to keep in the database. * Maximum number of entries per feed to keep in the database.
* *
* 0 to disable. * <p>0 to disable.
*/ */
@WithDefault("500") @WithDefault("500")
int maxFeedCapacity(); int maxFeedCapacity();
/** /**
* Limit the number of feeds a user can subscribe to. * Limit the number of feeds a user can subscribe to.
* *
* 0 to disable. * <p>0 to disable.
*/ */
@WithDefault("0") @WithDefault("0")
int maxFeedsPerUser(); int maxFeedsPerUser();
/** /** Rows to delete per query while cleaning up old entries. */
* Rows to delete per query while cleaning up old entries. @Positive
*/ @WithDefault("100")
@Positive int batchSize();
@WithDefault("100")
int batchSize();
/** /** Whether to keep starred entries when cleaning up old entries. */
* Whether to keep starred entries when cleaning up old entries. @WithDefault("true")
*/ boolean keepStarredEntries();
@WithDefault("true")
boolean keepStarredEntries();
default Instant statusesInstantThreshold() { default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null; return statusesMaxAge().toMillis() > 0
} ? Instant.now().minus(statusesMaxAge())
} : null;
} }
}
}
interface Users { interface Users {
/** /** Whether to let users create accounts for themselves. */
* Whether to let users create accounts for themselves. @WithDefault("false")
*/ boolean allowRegistrations();
@WithDefault("false")
boolean allowRegistrations();
/** /** Minimum password length for user accounts. */
* Minimum password length for user accounts. @WithDefault("4")
*/ int minimumPasswordLength();
@WithDefault("4")
int minimumPasswordLength();
/** /** Whether an email address is required when creating a user account. */
* Whether an email address is required when creating a user account. @WithDefault("false")
*/ boolean emailAddressRequired();
@WithDefault("false")
boolean emailAddressRequired();
/** /** Whether to create a demo account the first time the app starts. */
* Whether to create a demo account the first time the app starts. @WithDefault("false")
*/ boolean createDemoAccount();
@WithDefault("false") }
boolean createDemoAccount();
}
interface Websocket { interface Websocket {
/** /**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds. * Enable websocket connection so the server can notify web clients that there are new
*/ * entries for feeds.
@WithDefault("true") */
boolean enabled(); @WithDefault("true")
boolean enabled();
/** /**
* Interval at which the client will send a ping message on the websocket to keep the connection alive. * Interval at which the client will send a ping message on the websocket to keep the
*/ * connection alive.
@WithDefault("15m") */
Duration pingInterval(); @WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
/**
* If the websocket connection is disabled or the connection is lost, the client will reload
* the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
} }

2
commafeed-server/src/main/java/com/commafeed/CommaFeedConstants.java
View File

@@ -4,5 +4,5 @@ import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
public class CommaFeedConstants { public class CommaFeedConstants {
public static final String USERNAME_DEMO = "demo"; public static final String USERNAME_DEMO = "demo";
} }

27
commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
View File

@@ -1,25 +1,22 @@
package com.commafeed; package com.commafeed;
import java.time.InstantSource; import com.codahale.metrics.MetricRegistry;
import jakarta.enterprise.inject.Produces; import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import java.time.InstantSource;
import com.codahale.metrics.MetricRegistry;
@Singleton @Singleton
public class CommaFeedProducers { public class CommaFeedProducers {
@Produces @Produces
@Singleton @Singleton
public InstantSource instantSource() { public InstantSource instantSource() {
return InstantSource.system(); return InstantSource.system();
} }
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
} }

29
commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
View File

@@ -1,30 +1,27 @@
package com.commafeed; package com.commafeed;
import jakarta.inject.Singleton;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter; import lombok.Getter;
@Singleton @Singleton
@Getter @Getter
public class CommaFeedVersion { public class CommaFeedVersion {
private final String version; private final String version;
private final String gitCommit; private final String gitCommit;
public CommaFeedVersion() throws IOException { public CommaFeedVersion() throws IOException {
Properties properties = new Properties(); Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) { try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) { if (stream != null) {
properties.load(stream); properties.load(stream);
} }
} }
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
} }

60
commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
View File

@@ -1,52 +1,54 @@
package com.commafeed; package com.commafeed;
import com.commafeed.security.CookieService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import jakarta.annotation.Priority; import jakarta.annotation.Priority;
import jakarta.validation.ValidationException; import jakarta.validation.ValidationException;
import jakarta.ws.rs.core.NewCookie; import jakarta.ws.rs.core.NewCookie;
import jakarta.ws.rs.ext.Provider; import jakarta.ws.rs.ext.Provider;
import lombok.RequiredArgsConstructor;
import org.jboss.resteasy.reactive.RestResponse; import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder; import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
import org.jboss.resteasy.reactive.RestResponse.Status; import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper; import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import com.commafeed.security.CookieService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Provider @Provider
@Priority(1) @Priority(1)
public class ExceptionMappers { public class ExceptionMappers {
private final CookieService cookieService; private final CookieService cookieService;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class) @ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) { public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(Status.UNAUTHORIZED, new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations())); return RestResponse.status(
} Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class) @ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) { public RestResponse<AuthenticationFailed> authenticationFailed(
NewCookie logoutCookie = cookieService.buildLogoutCookie(); AuthenticationFailedException e) {
return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage())).cookie(logoutCookie).build(); NewCookie logoutCookie = cookieService.buildLogoutCookie();
} return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()))
.cookie(logoutCookie)
.build();
}
@ServerExceptionMapper(ValidationException.class) @ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) { public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage())); return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
} }
@RegisterForReflection @RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {} public record UnauthorizedResponse(String message, boolean allowRegistrations) {}
@RegisterForReflection @RegisterForReflection
public record AuthenticationFailed(String message) {} public record AuthenticationFailed(String message) {}
@RegisterForReflection @RegisterForReflection
public record ValidationFailed(String message) {} public record ValidationFailed(String message) {}
} }

28
commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
View File

@@ -1,29 +1,27 @@
package com.commafeed; package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule; import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer; import io.quarkus.jackson.ObjectMapperCustomizer;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
@Singleton @Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer { public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override @Override
public void customize(ObjectMapper objectMapper) { public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule()); objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds // read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true) objectMapper
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false) .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false); .configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics // add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false)); objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
} }
} }

544
commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
View File

@@ -6,221 +6,343 @@ import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection( @RegisterForReflection(
targets = { targets = {
// metrics // metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class, MetricRegistry.class,
Meter.class,
Gauge.class,
Counter.class,
Timer.class,
Histogram.class,
// rome // rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class, java.util.Date.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class, com.rometools.rome.feed.synd.SyndFeedImpl.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class, com.rometools.rome.feed.synd.SyndEntryImpl.class,
com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class,
com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class,
com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable // rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class, com.rometools.modules.activitystreams.types.Article.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class, com.rometools.modules.activitystreams.types.Bookmark.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class, com.rometools.modules.activitystreams.types.Event.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class, com.rometools.modules.activitystreams.types.Folder.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class, com.rometools.modules.activitystreams.types.Note.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class, com.rometools.modules.activitystreams.types.Photo.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class, com.rometools.modules.activitystreams.types.Place.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class, com.rometools.modules.activitystreams.types.Product.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class, com.rometools.modules.activitystreams.types.Service.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class, com.rometools.modules.activitystreams.types.Status.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class, com.rometools.modules.base.types.FloatUnit.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class, com.rometools.modules.base.types.IntUnit.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class, com.rometools.modules.base.types.ShippingType.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class, com.rometools.modules.base.types.Size.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class, com.rometools.modules.base.types.YearType.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class, com.rometools.modules.content.ContentItem.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class, com.rometools.modules.georss.geometries.Envelope.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class, com.rometools.modules.georss.geometries.LinearRing.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class, com.rometools.modules.georss.geometries.Polygon.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class, com.rometools.modules.georss.geometries.PositionList.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class, com.rometools.modules.mediarss.types.MediaGroup.class,
java.util.ArrayList.class, com.rometools.modules.mediarss.types.Metadata.class,
com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class,
com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class,
com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class,
com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class,
com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class,
com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class,
com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class,
com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class,
com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class,
com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class,
com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class,
com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class,
com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class,
com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class,
com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class,
com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class,
com.rometools.rome.feed.rss.Image.class,
com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class,
com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class,
com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules // rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class, com.rometools.modules.sse.modules.Conflict.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class, com.rometools.modules.cc.CreativeCommonsImpl.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class, com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class, com.rometools.modules.georss.SimpleModuleImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class, com.rometools.modules.itunes.EntryInformationImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class, com.rometools.modules.photocast.PhotocastModuleImpl.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class, com.rometools.modules.yahooweather.YWeatherModuleImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class, com.rometools.modules.sse.modules.Related.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class,
com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class,
com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class,
com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class,
com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class,
com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class,
com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class,
com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class,
com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library // extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class, com.rometools.rome.io.impl.RSS090Parser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class, com.rometools.rome.io.impl.RSS091UserlandParser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class, com.rometools.rome.io.impl.RSS093Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class, com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class, com.rometools.rome.io.impl.SyModuleParser.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class, com.rometools.rome.io.impl.RSS090Generator.class,
com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class, com.rometools.rome.io.impl.RSS091UserlandGenerator.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class, com.rometools.rome.io.impl.RSS093Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class, com.rometools.rome.io.impl.RSS10Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class, com.rometools.rome.io.impl.Atom10Generator.class,
com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class, com.rometools.rome.feed.synd.impl.ConverterForAtom10.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS092.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class, com.rometools.rome.feed.synd.impl.ConverterForRSS094.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.fyyd.io.FyydParser.class, com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.fyyd.io.FyydParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class, com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class, com.rometools.modules.cc.io.ModuleParserRSS1.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class, com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesGenerator.class, com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.content.io.ContentModuleGenerator.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class, com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class, com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, }) com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
public class NativeImageClasses { com.rometools.modules.cc.io.CCModuleGenerator.class,
} com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class,
com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class,
com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class,
com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML20.class,
})
public class NativeImageClasses {}

28
commafeed-server/src/main/java/com/commafeed/backend/Digests.java
View File

@@ -1,29 +1,27 @@
package com.commafeed.backend; package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction; import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import java.nio.charset.StandardCharsets;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
public class Digests { public class Digests {
public static String sha1Hex(byte[] input) { public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input); return hashBytesToHex(Hashing.sha1(), input);
} }
public static String sha1Hex(String input) { public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8)); return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
} }
public static String md5Hex(String input) { public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8)); return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
} }
private static String hashBytesToHex(HashFunction function, byte[] input) { private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString(); return function.hashBytes(input).toString();
} }
} }

246
commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java
View File

@@ -1,5 +1,12 @@
package com.commafeed.backend; package com.commafeed.backend;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.net.HttpHeaders;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressNetwork;
import inet.ipaddr.IPAddressString;
import jakarta.inject.Singleton;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.URI; import java.net.URI;
import java.net.UnknownHostException; import java.net.UnknownHostException;
@@ -8,9 +15,9 @@ import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.SequencedMap; import java.util.SequencedMap;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import lombok.RequiredArgsConstructor;
import jakarta.inject.Singleton; import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
import org.apache.hc.client5.http.DnsResolver; import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver; import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.ConnectionConfig;
@@ -35,137 +42,148 @@ import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout; import org.apache.hc.core5.util.Timeout;
import org.brotli.dec.BrotliInputStream; import org.brotli.dec.BrotliInputStream;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.net.HttpHeaders;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressNetwork;
import inet.ipaddr.IPAddressString;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
@Singleton @Singleton
@RequiredArgsConstructor @RequiredArgsConstructor
public class HttpClientFactory { public class HttpClientFactory {
private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE; private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress(); private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final CommaFeedVersion version; private final CommaFeedVersion version;
public CloseableHttpClient newClient(int poolSize) { public CloseableHttpClient newClient(int poolSize) {
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config, poolSize); PoolingHttpClientConnectionManager connectionManager =
String userAgent = config.httpClient() newConnectionManager(config, poolSize);
.userAgent() String userAgent =
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion())); config.httpClient()
return newClient(config, connectionManager, userAgent); .userAgent()
} .orElseGet(
() ->
String.format(
"CommaFeed/%s (https://github.com/Athou/commafeed)",
version.getVersion()));
return newClient(config, connectionManager, userAgent);
}
private CloseableHttpClient newClient(CommaFeedConfiguration config, HttpClientConnectionManager connectionManager, String userAgent) { private CloseableHttpClient newClient(
List<Header> headers = new ArrayList<>(); CommaFeedConfiguration config,
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en")); HttpClientConnectionManager connectionManager,
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache")); String userAgent) {
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache")); List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>(); SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>();
contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new); contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new); contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new); contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
RedirectStrategy redirectStrategy = config.httpClient().blockLocalAddresses() RedirectStrategy redirectStrategy =
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER) config.httpClient().blockLocalAddresses()
: new DefaultRedirectStrategy(); ? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
: new DefaultRedirectStrategy();
return HttpClientBuilder.create() return HttpClientBuilder.create()
.disableConnectionState() .disableConnectionState()
.useSystemProperties() .useSystemProperties()
.disableAutomaticRetries() .disableAutomaticRetries()
.disableCookieManagement() .disableCookieManagement()
.setUserAgent(userAgent) .setUserAgent(userAgent)
.setDefaultHeaders(headers) .setDefaultHeaders(headers)
.setConnectionManager(connectionManager) .setConnectionManager(connectionManager)
.evictExpiredConnections() .evictExpiredConnections()
.evictIdleConnections(TimeValue.of(config.httpClient().idleConnectionsEvictionInterval())) .evictIdleConnections(
.setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap)) TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.setRedirectStrategy(redirectStrategy) .setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
.build(); .setRedirectStrategy(redirectStrategy)
} .build();
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config, int poolSize) { private PoolingHttpClientConnectionManager newConnectionManager(
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build(); CommaFeedConfiguration config, int poolSize) {
DnsResolver dnsResolver = config.httpClient().blockLocalAddresses() ? new BlockLocalAddressesDnsResolver(DNS_RESOLVER) SSLFactory sslFactory =
: DNS_RESOLVER; SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
DnsResolver dnsResolver =
config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
: DNS_RESOLVER;
return PoolingHttpClientConnectionManagerBuilder.create() return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory)) .setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom() .setDefaultConnectionConfig(
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout())) ConnectionConfig.custom()
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout())) .setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive())) .setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.build()) .setTimeToLive(
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build()) Timeout.of(config.httpClient().connectionTimeToLive()))
.setMaxConnPerRoute(poolSize) .build())
.setMaxConnTotal(poolSize) .setDefaultTlsConfig(
.setDnsResolver(dnsResolver) TlsConfig.custom()
.build(); .setHandshakeTimeout(
Timeout.of(config.httpClient().sslHandshakeTimeout()))
.build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
} private static boolean isLocalAddress(InetAddress address) {
IPAddress ip = new IPAddressNetwork.IPAddressGenerator().from(address);
return ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip);
}
private static boolean isLocalAddress(InetAddress address) { private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver {
IPAddress ip = new IPAddressNetwork.IPAddressGenerator().from(address); @Override
return ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip); public InetAddress[] resolve(String host) throws UnknownHostException {
} InetAddress[] addresses = delegate.resolve(host);
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new UnknownHostException(
"Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
}
private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver { @Override
@Override public String resolveCanonicalHostname(String host) throws UnknownHostException {
public InetAddress[] resolve(String host) throws UnknownHostException { return delegate.resolveCanonicalHostname(host);
InetAddress[] addresses = delegate.resolve(host); }
for (InetAddress addr : addresses) { }
if (isLocalAddress(addr)) {
throw new UnknownHostException("Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
}
@Override @RequiredArgsConstructor
public String resolveCanonicalHostname(String host) throws UnknownHostException { private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
return delegate.resolveCanonicalHostname(host);
}
}
@RequiredArgsConstructor private final DnsResolver delegate;
private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
private final DnsResolver delegate; @Override
public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context)
throws HttpException {
URI redirectUri = super.getLocationURI(request, response, context);
@Override String host = redirectUri.getHost();
public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException { if (host == null) {
URI redirectUri = super.getLocationURI(request, response, context); throw new HttpException("Redirect URI does not have a host: " + redirectUri);
}
String host = redirectUri.getHost(); InetAddress[] addresses;
if (host == null) { try {
throw new HttpException("Redirect URI does not have a host: " + redirectUri); addresses = delegate.resolve(host);
} } catch (UnknownHostException e) {
throw new HttpException("Unknown host: " + host);
}
InetAddress[] addresses; for (InetAddress addr : addresses) {
try { if (isLocalAddress(addr)) {
addresses = delegate.resolve(host); throw new HttpException(
} catch (UnknownHostException e) { "Access to local address blocked: " + addr.getHostAddress());
throw new HttpException("Unknown host: " + host); }
} }
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new HttpException("Access to local address blocked: " + addr.getHostAddress());
}
}
return redirectUri;
}
}
return redirectUri;
}
}
} }

524
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -1,5 +1,15 @@
package com.commafeed.backend; package com.commafeed.backend;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI; import java.net.URI;
@@ -8,10 +18,12 @@ import java.time.Instant;
import java.time.InstantSource; import java.time.InstantSource;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import lombok.Builder;
import jakarta.inject.Singleton; import lombok.EqualsAndHashCode;
import jakarta.ws.rs.core.CacheControl; import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.config.RequestConfig;
@@ -27,282 +39,326 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.util.Timeout; import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate; import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry; /** Smart HTTP getter: handles gzip, ssl, last modified and etag headers */
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
@Singleton @Singleton
@Slf4j @Slf4j
public class HttpGetter { public class HttpGetter {
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final InstantSource instantSource; private final InstantSource instantSource;
private final CloseableHttpClient client; private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache; private final Cache<HttpRequest, HttpResponse> cache;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, HttpClientFactory httpClientFactory, public HttpGetter(
MetricRegistry metrics) { CommaFeedConfiguration config,
this.config = config; InstantSource instantSource,
this.instantSource = instantSource; HttpClientFactory httpClientFactory,
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads()); MetricRegistry metrics) {
this.cache = newCache(config); this.config = config;
this.instantSource = instantSource;
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size()); metrics.registerGauge(
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"), MetricRegistry.name(getClass(), "cache", "size"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> ArrayUtils.getLength(e.content)).sum()); () -> cache == null ? 0 : cache.size());
} metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() ->
cache == null
? 0
: cache.asMap().values().stream()
.mapToInt(e -> ArrayUtils.getLength(e.content))
.sum());
}
public HttpResult get(String url) throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException { public HttpResult get(String url)
return get(HttpRequest.builder(url).build()); throws IOException,
} NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request) public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException { throws IOException,
URI uri = URI.create(request.getUrl()); NotModifiedException,
ensureHttpScheme(uri.getScheme()); TooManyRequestsException,
SchemeNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
final HttpResponse response; final HttpResponse response;
if (cache == null) { if (cache == null) {
response = invoke(request); response = invoke(request);
} else { } else {
try { try {
response = cache.get(request, () -> invoke(request)); response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) { } catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) { if (e.getCause() instanceof IOException ioe) {
throw ioe; throw ioe;
} else { } else {
throw Lombok.sneakyThrow(e); throw Lombok.sneakyThrow(e);
} }
} }
} }
int code = response.code(); int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) { if (code == HttpStatus.SC_TOO_MANY_REQUESTS
throw new TooManyRequestsException(response.retryAfter()); || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
} throw new TooManyRequestsException(response.retryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) { if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received"); throw new NotModifiedException("'304 - not modified' http code received");
} }
if (code >= 300) { if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code); throw new HttpResponseException(code, "Server returned HTTP error code " + code);
} }
String lastModifiedHeader = response.lastModifiedHeader(); String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.eTagHeader(); String eTagHeader = response.eTagHeader();
Duration validFor = Optional.ofNullable(response.cacheControl()) Duration validFor =
.filter(cc -> cc.getMaxAge() >= 0) Optional.ofNullable(response.cacheControl())
.map(cc -> Duration.ofSeconds(cc.getMaxAge())) .filter(cc -> cc.getMaxAge() >= 0)
.orElse(Duration.ZERO); .map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.content(), response.contentType(), lastModifiedHeader, eTagHeader, response.urlAfterRedirect(), return new HttpResult(
validFor); response.content(),
} response.contentType(),
lastModifiedHeader,
eTagHeader,
response.urlAfterRedirect(),
validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException { private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) { if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
throw new SchemeNotAllowedException(scheme); throw new SchemeNotAllowedException(scheme);
} }
} }
private HttpResponse invoke(HttpRequest request) throws IOException { private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl()); log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create(); HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom() context.setRequestConfig(
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())) RequestConfig.custom()
// causes issues with some feeds .setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// see https://github.com/Athou/commafeed/issues/1572 // causes issues with some feeds
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344 // see https://github.com/Athou/commafeed/issues/1572
.setProtocolUpgradeEnabled(false) // and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.build()); .setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> { return client.execute(
byte[] content = resp.getEntity() == null ? null request.toClassicHttpRequest(),
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue()); context,
int code = resp.getCode(); resp -> {
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED)) byte[] content =
.map(NameValuePair::getValue) resp.getEntity() == null
.map(StringUtils::trimToNull) ? null
.orElse(null); : toByteArray(
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG)) resp.getEntity(),
.map(NameValuePair::getValue) config.httpClient().maxResponseSize().asLongValue());
.map(StringUtils::trimToNull) int code = resp.getCode();
.orElse(null); String lastModifiedHeader =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL)) CacheControl cacheControl =
.map(NameValuePair::getValue) Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(StringUtils::trimToNull) .map(NameValuePair::getValue)
.map(HttpGetter::toCacheControl) .map(StringUtils::trimToNull)
.orElse(null); .map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER)) Instant retryAfter =
.map(NameValuePair::getValue) Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(StringUtils::trimToNull) .map(NameValuePair::getValue)
.map(this::toInstant) .map(StringUtils::trimToNull)
.orElse(null); .map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null); String contentType =
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations()) Optional.ofNullable(resp.getEntity())
.map(RedirectLocations::getAll) .map(HttpEntity::getContentType)
.map(l -> Iterables.getLast(l, null)) .orElse(null);
.map(URI::toString) String urlAfterRedirect =
.orElse(request.getUrl()); Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect); return new HttpResponse(
}); code,
} lastModifiedHeader,
eTagHeader,
cacheControl,
retryAfter,
content,
contentType,
urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) { private static CacheControl toCacheControl(String headerValue) {
try { try {
return CacheControlDelegate.INSTANCE.fromString(headerValue); return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) { } catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue); log.debug("Invalid Cache-Control header: {}", headerValue);
return null; return null;
} }
} }
private Instant toInstant(String headerValue) { private Instant toInstant(String headerValue) {
if (headerValue == null) { if (headerValue == null) {
return null; return null;
} }
if (StringUtils.isNumeric(headerValue)) { if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue)); return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
} }
return DateUtils.parseStandardDate(headerValue); return DateUtils.parseStandardDate(headerValue);
} }
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException { private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) { if (entity.getContentLength() > maxBytes) {
throw new IOException( throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes)); "Response size (%s bytes) exceeds the maximum allowed size (%s bytes)"
} .formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) { try (InputStream input = entity.getContent()) {
if (input == null) { if (input == null) {
return null; return null;
} }
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes(); byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) { if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes)); throw new IOException(
} "Response size exceeds the maximum allowed size (%s bytes)"
return bytes; .formatted(maxBytes));
} }
} return bytes;
}
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) { private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache(); HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) { if (!cacheConfig.enabled()) {
return null; return null;
} }
return CacheBuilder.newBuilder() return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.content() != null ? value.content().length : 0) .weigher(
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue()) (HttpRequest key, HttpResponse value) ->
.expireAfterWrite(cacheConfig.expiration()) value.content() != null ? value.content().length : 0)
.build(); .maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
} .expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception { public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) { public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme); super("Scheme not allowed: " + scheme);
} }
} }
@Getter @Getter
public static class NotModifiedException extends Exception { public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
/** /** if the value of this header changed, this is its new value */
* if the value of this header changed, this is its new value private final String newLastModifiedHeader;
*/
private final String newLastModifiedHeader;
/** /** if the value of this header changed, this is its new value */
* if the value of this header changed, this is its new value private final String newEtagHeader;
*/
private final String newEtagHeader;
public NotModifiedException(String message) { public NotModifiedException(String message) {
this(message, null, null); this(message, null, null);
} }
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) { public NotModifiedException(
super(message); String message, String newLastModifiedHeader, String newEtagHeader) {
this.newLastModifiedHeader = newLastModifiedHeader; super(message);
this.newEtagHeader = newEtagHeader; this.newLastModifiedHeader = newLastModifiedHeader;
} this.newEtagHeader = newEtagHeader;
} }
}
@RequiredArgsConstructor @RequiredArgsConstructor
@Getter @Getter
public static class TooManyRequestsException extends Exception { public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private final Instant retryAfter; private final Instant retryAfter;
} }
@Getter @Getter
public static class HttpResponseException extends IOException { public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private final int code; private final int code;
public HttpResponseException(int code, String message) { public HttpResponseException(int code, String message) {
super(message); super(message);
this.code = code; this.code = code;
} }
} }
@Builder(builderMethodName = "") @Builder(builderMethodName = "")
@EqualsAndHashCode @EqualsAndHashCode
@Getter @Getter
public static class HttpRequest { public static class HttpRequest {
private String url; private String url;
private String lastModified; private String lastModified;
private String eTag; private String eTag;
public static HttpRequestBuilder builder(String url) { public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url); return new HttpRequestBuilder().url(url);
} }
public ClassicHttpRequest toClassicHttpRequest() { public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build(); ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) { if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified); req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
} }
if (eTag != null) { if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag); req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
} }
return req; return req;
} }
} }
private record HttpResponse(int code, String lastModifiedHeader, String eTagHeader, CacheControl cacheControl, Instant retryAfter, private record HttpResponse(
byte[] content, String contentType, String urlAfterRedirect) {} int code,
String lastModifiedHeader,
public record HttpResult(byte[] content, String contentType, String lastModifiedSince, String eTag, String urlAfterRedirect, String eTagHeader,
Duration validFor) {} CacheControl cacheControl,
Instant retryAfter,
byte[] content,
String contentType,
String urlAfterRedirect) {}
public record HttpResult(
byte[] content,
String contentType,
String lastModifiedSince,
String eTag,
String urlAfterRedirect,
Duration validFor) {}
} }

155
commafeed-server/src/main/java/com/commafeed/backend/Urls.java
View File

@@ -2,105 +2,104 @@ package com.commafeed.backend;
import java.net.URI; import java.net.URI;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Strings; import org.apache.commons.lang3.Strings;
import org.netpreserve.urlcanon.Canonicalizer; import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl; import org.netpreserve.urlcanon.ParsedUrl;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@UtilityClass @UtilityClass
@Slf4j @Slf4j
public class Urls { public class Urls {
private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?")); private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?"));
public static boolean isHttp(String url) { public static boolean isHttp(String url) {
return url.startsWith("http://"); return url.startsWith("http://");
} }
public static boolean isHttps(String url) { public static boolean isHttps(String url) {
return url.startsWith("https://"); return url.startsWith("https://");
} }
public static boolean isAbsolute(String url) { public static boolean isAbsolute(String url) {
return isHttp(url) || isHttps(url); return isHttp(url) || isHttps(url);
} }
/** /**
* * @param relativeUrl the url of the entry
* @param relativeUrl * @param feedLink the url of the feed as described in the feed
* the url of the entry * @param feedUrl the url of the feed that we used to fetch the feed
* @param feedLink * @return an absolute url pointing to the entry
* the url of the feed as described in the feed */
* @param feedUrl public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) {
* the url of the feed that we used to fetch the feed String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
* @return an absolute url pointing to the entry if (baseUrl == null) {
*/ return null;
public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) { }
String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try { try {
return URI.create(baseUrl).resolve(relativeUrl).toString(); return URI.create(baseUrl).resolve(relativeUrl).toString();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
log.debug("Unable to create absolute url from relative url: {} base: {}", relativeUrl, baseUrl, e); log.debug(
return null; "Unable to create absolute url from relative url: {} base: {}",
} relativeUrl,
} baseUrl,
e);
return null;
}
}
public static String removeTrailingSlash(String url) { public static String removeTrailingSlash(String url) {
if (url == null) { if (url == null) {
return null; return null;
} }
if (url.endsWith("/")) { if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1); url = url.substring(0, url.length() - 1);
} }
return url; return url;
} }
/** /**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates * Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to
*/ * identify a feed and avoid duplicates
public static String normalize(String url) { */
if (url == null) { public static String normalize(String url) {
return null; if (url == null) {
} return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url); ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl); Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString(); String normalized = parsedUrl.toString();
if (normalized == null) { if (normalized == null) {
normalized = url; normalized = url;
} }
// convert to lower case, the url probably won't work in some cases // convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid // after that but we don't care we just want to compare urls to avoid
// duplicates // duplicates
normalized = normalized.toLowerCase(); normalized = normalized.toLowerCase();
// store all urls as http // store all urls as http
if (normalized.startsWith("https")) { if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5); normalized = "http" + normalized.substring(5);
} }
// remove the www. part // remove the www. part
normalized = normalized.replace("//www.", "//"); normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner // feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com"); normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment // feedburner feeds have a special treatment
if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) { if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com"); normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = QUESTION_MARK.split(normalized)[0]; normalized = QUESTION_MARK.split(normalized)[0];
normalized = Strings.CS.removeEnd(normalized, "/"); normalized = Strings.CS.removeEnd(normalized, "/");
} }
return normalized; return normalized;
} }
} }

109
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
View File

@@ -1,71 +1,76 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory; import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory; import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser; import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate; import com.querydsl.core.types.Predicate;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Objects;
@Singleton @Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> { public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory; private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) { public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class); super(entityManager, FeedCategory.class);
} }
public List<FeedCategory> findAll(User user) { public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch(); return query().selectFrom(CATEGORY)
} .where(CATEGORY.user.eq(user))
.join(CATEGORY.user, QUser.user)
.fetchJoin()
.fetch();
}
public FeedCategory findById(User user, Long id) { public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne(); return query().selectFrom(CATEGORY)
} .where(CATEGORY.user.eq(user), CATEGORY.id.eq(id))
.fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) { public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate; Predicate parentPredicate;
if (parent == null) { if (parent == null) {
parentPredicate = CATEGORY.parent.isNull(); parentPredicate = CATEGORY.parent.isNull();
} else { } else {
parentPredicate = CATEGORY.parent.eq(parent); parentPredicate = CATEGORY.parent.eq(parent);
} }
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne(); return query().selectFrom(CATEGORY)
} .where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate)
.fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) { public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate; Predicate parentPredicate;
if (parent == null) { if (parent == null) {
parentPredicate = CATEGORY.parent.isNull(); parentPredicate = CATEGORY.parent.isNull();
} else { } else {
parentPredicate = CATEGORY.parent.eq(parent); parentPredicate = CATEGORY.parent.eq(parent);
} }
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch(); return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
} }
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) { public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList(); return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
} }
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
} }

93
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -1,64 +1,69 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.Strings;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed; import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription; import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions; import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.Strings;
@Singleton @Singleton
public class FeedDAO extends GenericDAO<Feed> { public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed; private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription; private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) { public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class); super(entityManager, Feed.class);
} }
public List<Feed> findByIds(List<Long> id) { public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch(); return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
} }
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) { public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED) JPAQuery<Feed> query =
.distinct() query().selectFrom(FEED)
// join on subscriptions to only refresh feeds that have subscribers .distinct()
.join(SUBSCRIPTION) // join on subscriptions to only refresh feeds that have subscribers
.on(SUBSCRIPTION.feed.eq(FEED)) .join(SUBSCRIPTION)
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now()))); .on(SUBSCRIPTION.feed.eq(FEED))
.where(
FEED.disabledUntil
.isNull()
.or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) { if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold)); query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
} }
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch(); return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
} }
public void setDisabledUntil(List<Long> feedIds, Instant date) { public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute(); updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
} }
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) { public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED) return query()
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash)) .selectFrom(FEED)
.fetch() .where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.stream() .fetch()
.filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl())) .stream()
.findFirst() .filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
.orElse(null); .findFirst()
} .orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) { public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription; QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch(); return query().selectFrom(FEED)
} .where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists())
.limit(max)
.fetch();
}
} }

48
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
View File

@@ -1,36 +1,38 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent; import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry; import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent; import com.commafeed.backend.model.QFeedEntryContent;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
@Singleton @Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> { public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent; private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry; private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) { public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class); super(entityManager, FeedEntryContent.class);
} }
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) { public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch(); return query().select(CONTENT)
} .from(CONTENT)
.where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash))
.fetch();
}
public long deleteWithoutEntries(int max) { public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id) List<Long> ids =
.from(CONTENT) query().select(CONTENT.id)
.leftJoin(ENTRY) .from(CONTENT)
.on(ENTRY.content.id.eq(CONTENT.id)) .leftJoin(ENTRY)
.where(ENTRY.id.isNull()) .on(ENTRY.content.id.eq(CONTENT.id))
.limit(max) .where(ENTRY.id.isNull())
.fetch(); .limit(max)
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute(); .fetch();
} return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
} }

144
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,14 +1,5 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry; import com.commafeed.backend.model.QFeedEntry;
@@ -16,83 +7,98 @@ import com.google.common.collect.Lists;
import com.querydsl.core.Tuple; import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression; import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Singleton @Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> { public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry; private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000; private static final int IN_CLAUSE_BATCH_SIZE = 1000;
public FeedEntryDAO(EntityManager entityManager) { public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class); super(entityManager, FeedEntry.class);
} }
public FeedEntry findExisting(String guidHash, Feed feed) { public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne(); return query().select(ENTRY)
} .from(ENTRY)
.where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed))
.limit(1)
.fetchOne();
}
public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) { public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
if (guidHashes.isEmpty()) { if (guidHashes.isEmpty()) {
return Set.of(); return Set.of();
} }
Set<String> result = new HashSet<>(); Set<String> result = new HashSet<>();
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) { for (List<String> batch :
result.addAll(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch)).fetch()); Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
} result.addAll(
return result; query().select(ENTRY.guidHash)
} .from(ENTRY)
.where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch))
.fetch());
}
return result;
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) { public List<FeedCapacity> findFeedsExceedingCapacity(
NumberExpression<Long> count = ENTRY.id.count(); long maxCapacity, long max, boolean keepStarredEntries) {
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY); NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);
if (keepStarredEntries) { if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY)); query.where(Predicates.isNotStarred(ENTRY));
} }
return query.groupBy(ENTRY.feed) return query.groupBy(ENTRY.feed).having(count.gt(maxCapacity)).limit(max).fetch().stream()
.having(count.gt(maxCapacity)) .map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
.limit(max) .toList();
.fetch() }
.stream()
.map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
.toList();
}
public int delete(Long feedId, long max) { public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch(); List<FeedEntry> list =
return delete(list); query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
} return delete(list);
}
/** /** Delete entries older than a certain date */
* Delete entries older than a certain date public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) {
*/ JPAQuery<FeedEntry> query =
public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) { query().selectFrom(ENTRY)
JPAQuery<FeedEntry> query = query().selectFrom(ENTRY) .where(ENTRY.published.lt(olderThan))
.where(ENTRY.published.lt(olderThan)) .orderBy(ENTRY.published.asc())
.orderBy(ENTRY.published.asc()) .limit(max);
.limit(max);
if (keepStarredEntries) { if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY)); query.where(Predicates.isNotStarred(ENTRY));
} }
return delete(query.fetch()); return delete(query.fetch());
} }
/** /** Delete the oldest entries of a feed */
* Delete the oldest entries of a feed public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
*/ JPAQuery<FeedEntry> query =
public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) { query().selectFrom(ENTRY)
JPAQuery<FeedEntry> query = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max); .where(ENTRY.feed.id.eq(feedId))
.orderBy(ENTRY.published.asc())
.limit(max);
if (keepStarredEntries) { if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY)); query.where(Predicates.isNotStarred(ENTRY));
} }
return delete(query.fetch()); return delete(query.fetch());
} }
public record FeedCapacity(Long id, Long capacity) {} public record FeedCapacity(Long id, Long capacity) {}
} }

492
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword; import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode; import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
@@ -32,273 +21,316 @@ import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.Expressions; import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.core.types.dsl.NumberExpression; import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
@Singleton @Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> { public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus; private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry; private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeed FEED = QFeed.feed; private static final QFeed FEED = QFeed.feed;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent; private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag; private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription; private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final FeedEntryTagDAO feedEntryTagDAO; private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) { public FeedEntryStatusDAO(
super(entityManager, FeedEntryStatus.class); EntityManager entityManager,
this.feedEntryTagDAO = feedEntryTagDAO; FeedEntryTagDAO feedEntryTagDAO,
this.config = config; CommaFeedConfiguration config) {
} super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) { public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch(); List<FeedEntryStatus> statuses =
FeedEntryStatus status = statuses.stream().findFirst().orElse(null); query().selectFrom(STATUS)
return handleStatus(user, status, sub, entry); .where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub))
} .fetch();
FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
return handleStatus(user, status, sub, entry);
}
/** /** creates an artificial "unread" status if status is null */
* creates an artificial "unread" status if status is null private FeedEntryStatus handleStatus(
*/ User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) { if (status == null) {
if (status == null) { Instant statusesInstantThreshold =
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold(); config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold); boolean read =
status = new FeedEntryStatus(user, sub, entry); statusesInstantThreshold != null
status.setRead(read); && entry.getPublished().isBefore(statusesInstantThreshold);
status.setMarkable(!read); status = new FeedEntryStatus(user, sub, entry);
} else { status.setRead(read);
status.setMarkable(true); status.setMarkable(!read);
} } else {
return status; status.setMarkable(true);
} }
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) { private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user, Map<Long, List<FeedEntryTag>> tagsByEntryIds =
statuses.stream().map(FeedEntryStatus::getEntry).toList()); feedEntryTagDAO.findByEntries(
for (FeedEntryStatus status : statuses) { user, statuses.stream().map(FeedEntryStatus::getEntry).toList());
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId()); for (FeedEntryStatus status : statuses) {
status.setTags(tags == null ? List.of() : tags); List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
} status.setTags(tags == null ? List.of() : tags);
} }
}
public List<FeedEntryStatus> findStarred(User user, List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, public List<FeedEntryStatus> findStarred(
ReadingOrder order, boolean includeContent) { User user,
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue()); List<FeedEntryKeyword> keywords,
if (includeContent || CollectionUtils.isNotEmpty(keywords)) { Instant newerThan,
query.join(STATUS.entry).fetchJoin(); int offset,
query.join(STATUS.entry.content, CONTENT).fetchJoin(); int limit,
} ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query =
query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) { if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords); applyKeywordsFilter(query, keywords);
} }
if (newerThan != null) { if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan)); query.where(STATUS.entryInserted.gt(newerThan));
} }
if (order == ReadingOrder.ASC) { if (order == ReadingOrder.ASC) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc()); query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else { } else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc()); query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
} }
if (offset > -1) { if (offset > -1) {
query.offset(offset); query.offset(offset);
} }
if (limit > -1) { if (limit > -1) {
query.limit(limit); query.limit(limit);
} }
setTimeout(query, config.database().queryTimeout()); setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch(); List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true)); statuses.forEach(s -> s.setMarkable(true));
if (includeContent) { if (includeContent) {
fetchTags(user, statuses); fetchTags(user, statuses);
} }
return statuses; return statuses;
} }
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly, public List<FeedEntryStatus> findBySubscriptions(
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent, User user,
String tag, Long minEntryId, Long maxEntryId) { List<FeedSubscription> subs,
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId())); boolean unreadOnly,
List<FeedEntryKeyword> keywords,
Instant newerThan,
int offset,
int limit,
ReadingOrder order,
boolean includeContent,
String tag,
Long minEntryId,
Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId =
subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY); JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs)); query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet())); query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) { if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin(); query.join(ENTRY.content, CONTENT).fetchJoin();
} }
if (CollectionUtils.isNotEmpty(keywords)) { if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords); applyKeywordsFilter(query, keywords);
} }
if (unreadOnly && tag == null) { if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate()); query.where(buildUnreadPredicate());
} }
if (tag != null) { if (tag != null) {
BooleanBuilder and = new BooleanBuilder(); BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId())); and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag)); and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and); query.join(ENTRY.tags, TAG).on(and);
} }
if (newerThan != null) { if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan)); query.where(ENTRY.inserted.goe(newerThan));
} }
if (minEntryId != null) { if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId)); query.where(ENTRY.id.gt(minEntryId));
} }
if (maxEntryId != null) { if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId)); query.where(ENTRY.id.lt(maxEntryId));
} }
if (order != null) { if (order != null) {
if (order == ReadingOrder.ASC) { if (order == ReadingOrder.ASC) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc()); query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else { } else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc()); query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
} }
} }
if (offset > -1) { if (offset > -1) {
query.offset(offset); query.offset(offset);
} }
if (limit > -1) { if (limit > -1) {
query.limit(limit); query.limit(limit);
} }
setTimeout(query, config.database().queryTimeout()); setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>(); List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch(); List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) { for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY); FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS); FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) { for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e)); statuses.add(handleStatus(user, s, sub, e));
} }
} }
if (includeContent) { if (includeContent) {
fetchTags(user, statuses); fetchTags(user, statuses);
} }
return statuses; return statuses;
} }
private void applyKeywordsFilter(JPAQuery<?> query, List<FeedEntryKeyword> keywords) { private void applyKeywordsFilter(JPAQuery<?> query, List<FeedEntryKeyword> keywords) {
for (FeedEntryKeyword keyword : keywords) { for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder(); BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.keyword())); or.or(CONTENT.content.containsIgnoreCase(keyword.keyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.keyword())); or.or(CONTENT.title.containsIgnoreCase(keyword.keyword()));
if (keyword.mode() == Mode.EXCLUDE) { if (keyword.mode() == Mode.EXCLUDE) {
or.not(); or.not();
} }
query.where(or); query.where(or);
} }
} }
public UnreadCount getUnreadCount(FeedSubscription sub) { public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max()) JPAQuery<Tuple> query =
.from(ENTRY) query().select(ENTRY.count(), ENTRY.published.max())
.leftJoin(ENTRY.statuses, STATUS) .from(ENTRY)
.on(STATUS.subscription.eq(sub)) .leftJoin(ENTRY.statuses, STATUS)
.where(ENTRY.feed.eq(sub.getFeed())) .on(STATUS.subscription.eq(sub))
.where(buildUnreadPredicate()); .where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne(); Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count()); Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max()); Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published); return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
} }
private BooleanBuilder buildUnreadPredicate() { private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder(); BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull()); or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse()); or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold(); Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) { if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold)); return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else { } else {
return or; return or;
} }
} }
public long deleteOldStatuses(Instant olderThan, int limit) { public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id) List<Long> ids =
.from(STATUS) query().select(STATUS.id)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse()) .from(STATUS)
.limit(limit) .where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.fetch(); .limit(limit)
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute(); .fetch();
} return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
public long autoMarkAsRead(int limit) { public long autoMarkAsRead(int limit) {
Instant now = Instant.now(); Instant now = Instant.now();
BooleanBuilder where = new BooleanBuilder(); BooleanBuilder where = new BooleanBuilder();
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull()); where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull());
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0)); where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
NumberExpression<Integer> daysDiff = Expressions.numberTemplate(Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, NumberExpression<Integer> daysDiff =
now); Expressions.numberTemplate(
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays)); Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, now);
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
where.and(buildUnreadPredicate()); where.and(buildUnreadPredicate());
List<Tuple> tuples = query().select(ENTRY, STATUS, SUBSCRIPTION) List<Tuple> tuples =
.from(ENTRY) query().select(ENTRY, STATUS, SUBSCRIPTION)
.join(ENTRY.feed, FEED) .from(ENTRY)
.join(SUBSCRIPTION) .join(ENTRY.feed, FEED)
.on(SUBSCRIPTION.feed.eq(FEED)) .join(SUBSCRIPTION)
.leftJoin(ENTRY.statuses, STATUS) .on(SUBSCRIPTION.feed.eq(FEED))
.on(STATUS.subscription.eq(SUBSCRIPTION)) .leftJoin(ENTRY.statuses, STATUS)
.where(where) .on(STATUS.subscription.eq(SUBSCRIPTION))
.limit(limit) .where(where)
.fetch(); .limit(limit)
.fetch();
long updated = 0; long updated = 0;
// Update existing statuses // Update existing statuses
List<Long> statusIdsToUpdate = tuples.stream() List<Long> statusIdsToUpdate =
.map(t -> t.get(STATUS)) tuples.stream()
.filter(s -> s != null && s.getId() != null) .map(t -> t.get(STATUS))
.map(FeedEntryStatus::getId) .filter(s -> s != null && s.getId() != null)
.distinct() .map(FeedEntryStatus::getId)
.toList(); .distinct()
.toList();
if (!statusIdsToUpdate.isEmpty()) { if (!statusIdsToUpdate.isEmpty()) {
updated += updateQuery(STATUS).where(STATUS.id.in(statusIdsToUpdate)).set(STATUS.read, true).execute(); updated +=
} updateQuery(STATUS)
.where(STATUS.id.in(statusIdsToUpdate))
.set(STATUS.read, true)
.execute();
}
// Insert new statuses for entries without existing status // Insert new statuses for entries without existing status
for (Tuple tuple : tuples) { for (Tuple tuple : tuples) {
FeedEntryStatus status = tuple.get(STATUS); FeedEntryStatus status = tuple.get(STATUS);
if (status == null || status.getId() == null) { if (status == null || status.getId() == null) {
FeedEntry entry = tuple.get(ENTRY); FeedEntry entry = tuple.get(ENTRY);
FeedSubscription sub = tuple.get(SUBSCRIPTION); FeedSubscription sub = tuple.get(SUBSCRIPTION);
FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry); FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry);
newStatus.setRead(true); newStatus.setRead(true);
persist(newStatus); persist(newStatus);
updated++; updated++;
} }
} }
return updated;
}
return updated;
}
} }

47
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
View File

@@ -1,39 +1,38 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag; import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag; import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Singleton @Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> { public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag; private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) { public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class); super(entityManager, FeedEntryTag.class);
} }
public List<String> findByUser(User user) { public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch(); return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
} }
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) { public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch(); return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
} }
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) { public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG) return query()
.where(TAG.user.eq(user), TAG.entry.in(entries)) .selectFrom(TAG)
.fetch() .where(TAG.user.eq(user), TAG.entry.in(entries))
.stream() .fetch()
.collect(Collectors.groupingBy(t -> t.getEntry().getId())); .stream()
} .collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
} }

216
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
View File

@@ -1,20 +1,5 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel; import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory; import com.commafeed.backend.model.FeedCategory;
@@ -23,109 +8,136 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription; import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.querydsl.jpa.JPQLQuery; import com.querydsl.jpa.JPQLQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
@Singleton @Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> { public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription; private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager; private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) { public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class); super(entityManager, FeedSubscription.class);
this.entityManager = entityManager; this.entityManager = entityManager;
} }
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) { public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class) entityManager
.getFactory() .unwrap(SharedSessionContractImplementor.class)
.getServiceRegistry() .getFactory()
.getService(EventListenerRegistry.class) .getServiceRegistry()
.getEventListenerGroup(EventType.POST_COMMIT_INSERT) .getService(EventListenerRegistry.class)
.appendListener(new PostCommitInsertEventListener() { .getEventListenerGroup(EventType.POST_COMMIT_INSERT)
@Override .appendListener(
public void onPostInsert(PostInsertEvent event) { new PostCommitInsertEventListener() {
if (event.getEntity() instanceof FeedSubscription s) { @Override
consumer.accept(s); public void onPostInsert(PostInsertEvent event) {
} if (event.getEntity() instanceof FeedSubscription s) {
} consumer.accept(s);
}
}
@Override @Override
public boolean requiresPostCommitHandling(EntityPersister persister) { public boolean requiresPostCommitHandling(EntityPersister persister) {
return true; return true;
} }
@Override @Override
public void onPostInsertCommitFailed(PostInsertEvent event) { public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing // do nothing
} }
}); });
} }
public FeedSubscription findById(User user, Long id) { public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION) List<FeedSubscription> subs =
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id)) query().selectFrom(SUBSCRIPTION)
.leftJoin(SUBSCRIPTION.feed) .where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.fetchJoin() .leftJoin(SUBSCRIPTION.feed)
.leftJoin(SUBSCRIPTION.category) .fetchJoin()
.fetchJoin() .leftJoin(SUBSCRIPTION.category)
.fetch(); .fetchJoin()
FeedSubscription sub = subs.stream().findFirst().orElse(null); .fetch();
return initRelations(sub); FeedSubscription sub = subs.stream().findFirst().orElse(null);
} return initRelations(sub);
}
public List<FeedSubscription> findByFeed(Feed feed) { public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch(); return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
} }
public FeedSubscription findByFeed(User user, Feed feed) { public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION) List<FeedSubscription> subs =
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed)) query().selectFrom(SUBSCRIPTION)
.fetch(); .where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
FeedSubscription sub = subs.stream().findFirst().orElse(null); .fetch();
return initRelations(sub); FeedSubscription sub = subs.stream().findFirst().orElse(null);
} return initRelations(sub);
}
public List<FeedSubscription> findAll(User user) { public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION) List<FeedSubscription> subs =
.where(SUBSCRIPTION.user.eq(user)) query().selectFrom(SUBSCRIPTION)
.leftJoin(SUBSCRIPTION.feed) .where(SUBSCRIPTION.user.eq(user))
.fetchJoin() .leftJoin(SUBSCRIPTION.feed)
.leftJoin(SUBSCRIPTION.category) .fetchJoin()
.fetchJoin() .leftJoin(SUBSCRIPTION.category)
.fetch(); .fetchJoin()
return initRelations(subs); .fetch();
} return initRelations(subs);
}
public Long count(User user) { public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne(); return query().select(SUBSCRIPTION.count())
} .from(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) { public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)); JPQLQuery<FeedSubscription> query =
if (category == null) { query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
query.where(SUBSCRIPTION.category.isNull()); if (category == null) {
} else { query.where(SUBSCRIPTION.category.isNull());
query.where(SUBSCRIPTION.category.eq(category)); } else {
} query.where(SUBSCRIPTION.category.eq(category));
return initRelations(query.fetch()); }
} return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) { public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet()); Set<Long> categoryIds =
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList(); categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
} return findAll(user).stream()
.filter(
s ->
s.getCategory() != null
&& categoryIds.contains(s.getCategory().getId()))
.toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) { private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations); list.forEach(this::initRelations);
return list; return list;
} }
private FeedSubscription initRelations(FeedSubscription sub) { private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) { if (sub != null) {
Models.initialize(sub.getFeed()); Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory()); Models.initialize(sub.getCategory());
} }
return sub; return sub;
} }
} }

81
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
View File

@@ -1,66 +1,61 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel; import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath; import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause; import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory; import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause; import com.querydsl.jpa.impl.JPAUpdateClause;
import jakarta.persistence.EntityManager;
import java.time.Duration;
import java.util.Collection;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.hibernate.jpa.SpecHints;
@RequiredArgsConstructor @RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> { public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager; private final EntityManager entityManager;
private final Class<T> entityClass; private final Class<T> entityClass;
protected JPAQueryFactory query() { protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager); return new JPAQueryFactory(entityManager);
} }
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) { protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath); return new JPAUpdateClause(entityManager, entityPath);
} }
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) { protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath); return new JPADeleteClause(entityManager, entityPath);
} }
public void persist(T model) { public void persist(T model) {
entityManager.persist(model); entityManager.persist(model);
} }
public T merge(T model) { public T merge(T model) {
return entityManager.merge(model); return entityManager.merge(model);
} }
public T findById(Long id) { public T findById(Long id) {
return entityManager.find(entityClass, id); return entityManager.find(entityClass, id);
} }
public void delete(T object) { public void delete(T object) {
if (object != null) { if (object != null) {
entityManager.remove(object); entityManager.remove(object);
} }
} }
public int delete(Collection<T> objects) { public int delete(Collection<T> objects) {
objects.forEach(this::delete); objects.forEach(this::delete);
return objects.size(); return objects.size();
} }
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
} }

12
commafeed-server/src/main/java/com/commafeed/backend/dao/Predicates.java
View File

@@ -4,15 +4,17 @@ import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryStatus; import com.commafeed.backend.model.QFeedEntryStatus;
import com.querydsl.core.types.dsl.BooleanExpression; import com.querydsl.core.types.dsl.BooleanExpression;
import com.querydsl.jpa.JPAExpressions; import com.querydsl.jpa.JPAExpressions;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
public class Predicates { public class Predicates {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus; private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
public static BooleanExpression isNotStarred(QFeedEntry entry) { public static BooleanExpression isNotStarred(QFeedEntry entry) {
return JPAExpressions.selectOne().from(STATUS).where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue())).notExists(); return JPAExpressions.selectOne()
} .from(STATUS)
.where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue()))
.notExists();
}
} }

18
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
View File

@@ -1,19 +1,17 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction; import io.quarkus.narayana.jta.QuarkusTransaction;
import jakarta.inject.Singleton;
import java.util.concurrent.Callable;
@Singleton @Singleton
public class UnitOfWork { public class UnitOfWork {
public void run(Runnable runnable) { public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable); QuarkusTransaction.joiningExisting().run(runnable);
} }
public <T> T call(Callable<T> callable) { public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable); return QuarkusTransaction.joiningExisting().call(callable);
} }
} }

37
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
View File

@@ -1,33 +1,32 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser; import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton @Singleton
public class UserDAO extends GenericDAO<User> { public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user; private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) { public UserDAO(EntityManager entityManager) {
super(entityManager, User.class); super(entityManager, User.class);
} }
public User findByName(String name) { public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne(); return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
} }
public User findByApiKey(String key) { public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne(); return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
} }
public User findByEmail(String email) { public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne(); return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
} }
public long count() { public long count() {
return query().select(USER.count()).from(USER).fetchOne(); return query().select(USER.count()).from(USER).fetchOne();
} }
} }

44
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
View File

@@ -1,39 +1,37 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole; import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole; import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role; import com.commafeed.backend.model.UserRole.Role;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Singleton @Singleton
public class UserRoleDAO extends GenericDAO<UserRole> { public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole; private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) { public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class); super(entityManager, UserRole.class);
} }
public List<UserRole> findAll() { public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch(); return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
} }
public List<UserRole> findAll(User user) { public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch(); return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
} }
public Set<Role> findRoles(User user) { public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet()); return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
} }
public long countAdmins() { public long countAdmins() {
return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne(); return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne();
} }
} }

19
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
View File

@@ -1,22 +1,21 @@
package com.commafeed.backend.dao; package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings; import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings; import com.commafeed.backend.model.UserSettings;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton @Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> { public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings; private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) { public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class); super(entityManager, UserSettings.class);
} }
public UserSettings findByUser(User user) { public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst(); return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
} }
} }

89
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -1,21 +1,17 @@
package com.commafeed.backend.favicon; package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -23,44 +19,49 @@ import lombok.extern.slf4j.Slf4j;
@Priority(3) @Priority(3)
public class FacebookFaviconFetcher implements FaviconFetcher { public class FacebookFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter; private final HttpGetter getter;
@Override @Override
public Favicon fetch(Feed feed) { public Favicon fetch(Feed feed) {
String url = feed.getUrl(); String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) { if (!url.toLowerCase().contains("www.facebook.com")) {
return null; return null;
} }
String userName = extractUserName(url); String userName = extractUserName(url);
if (userName == null) { if (userName == null) {
return null; return null;
} }
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName); String iconUrl =
String.format(
"https://graph.facebook.com/%s/picture?type=square&height=16", userName);
try { try {
log.debug("Getting Facebook user's icon, {}", url); log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl); HttpResult iconResult = getter.get(iconUrl);
return new Favicon(iconResult.content(), iconResult.contentType()); return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e); log.debug("Failed to retrieve Facebook icon", e);
return null; return null;
} }
} }
private String extractUserName(String url) { private String extractUserName(String url) {
URI uri; URI uri;
try { try {
uri = new URI(url); uri = new URI(url);
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
log.debug("could not parse url", e); log.debug("could not parse url", e);
return null; return null;
} }
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream()
.filter(p -> "id".equals(p.getName()))
.map(NameValuePair::getValue)
.findFirst()
.orElse(null);
}
} }

28
commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
View File

@@ -1,24 +1,24 @@
package com.commafeed.backend.favicon; package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public record Favicon(byte[] icon, MediaType mediaType) { public record Favicon(byte[] icon, MediaType mediaType) {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon"); private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
public Favicon(byte[] icon, String contentType) { public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType)); this(icon, parseMediaType(contentType));
} }
private static MediaType parseMediaType(String contentType) { private static MediaType parseMediaType(String contentType) {
try { try {
return MediaType.valueOf(contentType); return MediaType.valueOf(contentType);
} catch (Exception e) { } catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType, e); log.debug(
return DEFAULT_MEDIA_TYPE; "invalid content type '{}' received, returning default value", contentType, e);
} return DEFAULT_MEDIA_TYPE;
} }
} }
}

3
commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java
View File

@@ -4,6 +4,5 @@ import com.commafeed.backend.model.Feed;
public interface FaviconFetcher { public interface FaviconFetcher {
Favicon fetch(Feed feed); Favicon fetch(Feed feed);
} }

40
commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java
View File

@@ -1,39 +1,35 @@
package com.commafeed.backend.favicon; package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
/** /** Fetch favicon from the url declared in the feed. */
* Fetch favicon from the url declared in the feed.
*/
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
@Priority(2) @Priority(2)
public class FeedFaviconFetcher implements FaviconFetcher { public class FeedFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter; private final HttpGetter getter;
@Override @Override
public Favicon fetch(Feed feed) { public Favicon fetch(Feed feed) {
String url = feed.getIconUrl(); String url = feed.getIconUrl();
if (url == null) { if (url == null) {
return null; return null;
} }
try { try {
HttpResult result = getter.get(url); HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType()); return new Favicon(result.content(), result.contentType());
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e); log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null; return null;
} }
} }
} }

83
commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java
View File

@@ -1,65 +1,60 @@
package com.commafeed.backend.favicon; package com.commafeed.backend.favicon;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority; import jakarta.annotation.Priority;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter; /** Extracts favicon url from html page. */
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Extracts favicon url from html page.
*/
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
@Priority(1) @Priority(1)
public class HtmlFaviconFetcher implements FaviconFetcher { public class HtmlFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter; private final HttpGetter getter;
@Override @Override
public Favicon fetch(Feed feed) { public Favicon fetch(Feed feed) {
String url = feed.getLink(); String url = feed.getLink();
if (url == null) { if (url == null) {
return null; return null;
} }
Document doc; Document doc;
try { try {
HttpResult result = getter.get(url); HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url); doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e); log.debug("Failed to retrieve page to find icon", e);
return null; return null;
} }
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]"); Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) { if (icons.isEmpty()) {
log.debug("No icon found in page {}", url); log.debug("No icon found in page {}", url);
return null; return null;
} }
String href = icons.getFirst().attr("abs:href"); String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) { if (StringUtils.isBlank(href)) {
log.debug("No icon found in page"); log.debug("No icon found in page");
return null; return null;
} }
try { try {
HttpResult result = getter.get(href); HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType()); return new Favicon(result.content(), result.contentType());
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e); log.debug("Failed to retrieve icon found in page {}", href, e);
return null; return null;
} }
} }
} }

49
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java
View File

@@ -1,45 +1,40 @@
package com.commafeed.backend.favicon; package com.commafeed.backend.favicon;
import java.net.URI;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import java.net.URI;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
/** /** Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico) */
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
*/
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
@Priority(0) @Priority(0)
public class RootFaviconFetcher implements FaviconFetcher { public class RootFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter; private final HttpGetter getter;
@Override @Override
public Favicon fetch(Feed feed) { public Favicon fetch(Feed feed) {
String url = feed.getLink(); String url = feed.getLink();
if (url == null) { if (url == null) {
url = feed.getUrl(); url = feed.getUrl();
} }
try { try {
URI uri = URI.create(url.trim()); URI uri = URI.create(url.trim());
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost()); String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
log.debug("getting root icon at {}", faviconUrl); log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl); HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType()); return new Favicon(result.content(), result.contentType());
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e); log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null; return null;
} }
} }
} }

210
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -1,18 +1,5 @@
package com.commafeed.backend.favicon; package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.HttpResult;
@@ -23,9 +10,18 @@ import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@@ -33,99 +29,121 @@ import lombok.extern.slf4j.Slf4j;
@Priority(3) @Priority(3)
public class YoutubeFaviconFetcher implements FaviconFetcher { public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final String PART_SNIPPET = "snippet"; private static final String PART_SNIPPET = "snippet";
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url"); private static final JsonPointer CHANNEL_THUMBNAIL_URL =
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId"); JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID =
JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter; private final HttpGetter getter;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper; private final ObjectMapper objectMapper;
@Override @Override
public Favicon fetch(Feed feed) { public Favicon fetch(Feed feed) {
String url = feed.getUrl(); String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) { if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null; return null;
} }
Optional<String> googleAuthKey = config.googleAuthKey(); Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) { if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured"); log.debug("no google auth key configured");
return null; return null;
} }
try { try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams(); List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst(); Optional<NameValuePair> userId =
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst(); params.stream()
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst(); .filter(nvp -> nvp.getName().equalsIgnoreCase("user"))
.findFirst();
Optional<NameValuePair> channelId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id"))
.findFirst();
Optional<NameValuePair> playlistId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id"))
.findFirst();
byte[] response = null; byte[] response = null;
if (userId.isPresent()) { if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue()); log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue()); response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) { } else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue()); log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue()); response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) { } else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue()); log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue()); response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
} }
if (ArrayUtils.isEmpty(response)) { if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response"); log.debug("youtube api returned empty response");
return null; return null;
} }
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL); JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) { if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response"); log.debug("youtube api returned invalid response");
return null; return null;
} }
HttpResult iconResult = getter.get(thumbnailUrl.asText()); HttpResult iconResult = getter.get(thumbnailUrl.asText());
return new Favicon(iconResult.content(), iconResult.contentType()); return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e); log.debug("Failed to retrieve YouTube icon", e);
return null; return null;
} }
} }
private byte[] fetchForUser(String googleAuthKey, String userId) private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException { throws IOException,
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels") NotModifiedException,
.queryParam("part", PART_SNIPPET) TooManyRequestsException,
.queryParam("key", googleAuthKey) SchemeNotAllowedException {
.queryParam("forUsername", userId) URI uri =
.build(); UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
return getter.get(uri.toString()).content(); .queryParam("part", PART_SNIPPET)
} .queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId) private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException { throws IOException,
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels") NotModifiedException,
.queryParam("part", PART_SNIPPET) TooManyRequestsException,
.queryParam("key", googleAuthKey) SchemeNotAllowedException {
.queryParam("id", channelId) URI uri =
.build(); UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
return getter.get(uri.toString()).content(); .queryParam("part", PART_SNIPPET)
} .queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId) private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException { throws IOException,
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists") NotModifiedException,
.queryParam("part", PART_SNIPPET) TooManyRequestsException,
.queryParam("key", googleAuthKey) SchemeNotAllowedException {
.queryParam("id", playlistId) URI uri =
.build(); UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
byte[] playlistBytes = getter.get(uri.toString()).content(); .queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).content();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID); JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) { if (channelId.isMissingNode()) {
return new byte[0]; return new byte[0];
} }
return fetchForChannel(googleAuthKey, channelId.asText());
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
} }

40
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
View File

@@ -2,30 +2,28 @@ package com.commafeed.backend.feed;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
/** /** A keyword used in a search query */
* A keyword used in a search query
*/
public record FeedEntryKeyword(String keyword, Mode mode) { public record FeedEntryKeyword(String keyword, Mode mode) {
public enum Mode { public enum Mode {
INCLUDE, EXCLUDE INCLUDE,
} EXCLUDE
}
public static List<FeedEntryKeyword> fromQueryString(String keywords) { public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>(); List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) { if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) { for (String keyword : StringUtils.split(keywords)) {
boolean not = false; boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) { if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true; not = true;
keyword = keyword.substring(1); keyword = keyword.substring(1);
} }
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE)); list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
} }
} }
return list; return list;
} }
} }

196
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
import com.commafeed.backend.Digests; import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpRequest; import com.commafeed.backend.HttpGetter.HttpRequest;
@@ -22,96 +11,139 @@ import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException; import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult; import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider; import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All; import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
/** /** Fetches a feed then parses it */
* Fetches a feed then parses it
*/
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedFetcher { public class FeedFetcher {
private final FeedParser parser; private final FeedParser parser;
private final HttpGetter getter; private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders; private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) { public FeedFetcher(
this.parser = parser; FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.getter = getter; this.parser = parser;
this.urlProviders = urlProviders; this.getter = getter;
} this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, public FeedFetcherResult fetch(
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException, String feedUrl,
TooManyRequestsException, SchemeNotAllowedException, NoFeedFoundException { boolean extractFeedUrlFromHtml,
log.debug("Fetching feed {}", feedUrl); String lastModified,
String eTag,
Instant lastPublishedDate,
String lastContentHash)
throws FeedParsingException,
IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException,
NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build()); HttpResult result =
byte[] content = result.content(); getter.get(
HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.content();
FeedParserResult parserResult; FeedParserResult parserResult;
try { try {
parserResult = parser.parse(result.urlAfterRedirect(), content); parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) { } catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) { if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8)); String extractedUrl =
if (StringUtils.isNotBlank(extractedUrl)) { extractFeedUrl(
feedUrl = extractedUrl; urlProviders,
feedUrl,
new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build()); result =
content = result.content(); getter.get(
parserResult = parser.parse(result.urlAfterRedirect(), content); HttpRequest.builder(extractedUrl)
} else { .lastModified(lastModified)
throw new NoFeedFoundException(e); .eTag(eTag)
} .build());
} else { content = result.content();
throw e; parserResult = parser.parse(result.urlAfterRedirect(), content);
} } else {
} throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) { if (content == null) {
throw new IOException("Feed content is empty."); throw new IOException("Feed content is empty.");
} }
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.lastModifiedSince()); boolean lastModifiedHeaderValueChanged =
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag()); !Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
String hash = Digests.sha1Hex(content); String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) { if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl); log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null, throw new NotModifiedException(
etagHeaderValueChanged ? result.eTag() : null); "content hash not modified",
} lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) { if (lastPublishedDate != null
log.debug("publishedDate not modified: {}", feedUrl); && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
throw new NotModifiedException("publishedDate not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null, log.debug("publishedDate not modified: {}", feedUrl);
etagHeaderValueChanged ? result.eTag() : null); throw new NotModifiedException(
} "publishedDate not modified",
lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
return new FeedFetcherResult(parserResult, result.urlAfterRedirect(), result.lastModifiedSince(), result.eTag(), hash, return new FeedFetcherResult(
result.validFor()); parserResult,
} result.urlAfterRedirect(),
result.lastModifiedSince(),
result.eTag(),
hash,
result.validFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) { private static String extractFeedUrl(
return urlProviders.stream() List<FeedURLProvider> urlProviders, String url, String urlContent) {
.flatMap(provider -> provider.get(url, urlContent).stream()) return urlProviders.stream()
.filter(StringUtils::isNotBlank) .flatMap(provider -> provider.get(url, urlContent).stream())
.findFirst() .filter(StringUtils::isNotBlank)
.orElse(null); .findFirst()
} .orElse(null);
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader, public record FeedFetcherResult(
String contentHash, Duration validFor) {} FeedParserResult feed,
String urlAfterRedirect,
String lastModifiedHeader,
String lastETagHeader,
String contentHash,
Duration validFor) {}
public static class NoFeedFoundException extends Exception { public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
} }

468
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,19 +1,5 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge; import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
@@ -25,229 +11,297 @@ import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedRefreshEngine { public class FeedRefreshEngine {
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO; private final FeedDAO feedDAO;
private final FeedRefreshWorker worker; private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater; private final FeedRefreshUpdater updater;
private final FeedUpdateNotifier notifier; private final FeedUpdateNotifier notifier;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final Meter refill; private final Meter refill;
private final BlockingDeque<Feed> queue; private final BlockingDeque<Feed> queue;
private ExecutorService feedProcessingLoopExecutor; private ExecutorService feedProcessingLoopExecutor;
private ExecutorService refillLoopExecutor; private ExecutorService refillLoopExecutor;
private ThreadPoolExecutor refillExecutor; private ThreadPoolExecutor refillExecutor;
private ThreadPoolExecutor workerExecutor; private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor databaseUpdaterExecutor; private ThreadPoolExecutor databaseUpdaterExecutor;
private ThreadPoolExecutor notifierExecutor; private ThreadPoolExecutor notifierExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater, public FeedRefreshEngine(
FeedUpdateNotifier notifier, CommaFeedConfiguration config, MetricRegistry metrics) { UnitOfWork unitOfWork,
this.unitOfWork = unitOfWork; FeedDAO feedDAO,
this.feedDAO = feedDAO; FeedRefreshWorker worker,
this.worker = worker; FeedRefreshUpdater updater,
this.updater = updater; FeedUpdateNotifier notifier,
this.notifier = notifier; CommaFeedConfiguration config,
this.config = config; MetricRegistry metrics) {
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill")); this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.notifier = notifier;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>(); this.queue = new LinkedBlockingDeque<>();
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size); metrics.register(
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) () -> workerExecutor.getActiveCount()); MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), metrics.register(
(Gauge<Integer>) () -> databaseUpdaterExecutor.getActiveCount()); MetricRegistry.name(getClass(), "worker", "active"),
metrics.register(MetricRegistry.name(getClass(), "notifier", "active"), (Gauge<Integer>) () -> notifierExecutor.getActiveCount()); (Gauge<Integer>) () -> workerExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "notifier", "queue"), (Gauge<Integer>) () -> notifierExecutor.getQueue().size()); metrics.register(
} MetricRegistry.name(getClass(), "updater", "active"),
(Gauge<Integer>) () -> databaseUpdaterExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "notifier", "active"),
(Gauge<Integer>) () -> notifierExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "notifier", "queue"),
(Gauge<Integer>) () -> notifierExecutor.getQueue().size());
}
private void createExecutors() { private void createExecutors() {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor(); this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor(); this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService(); this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads()); this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads()); this.databaseUpdaterExecutor =
this.notifierExecutor = newDiscardingExecutorService(config.pushNotifications().threads(), newBlockingExecutorService(config.feedRefresh().databaseThreads());
config.pushNotifications().queueCapacity()); this.notifierExecutor =
} newDiscardingExecutorService(
config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
}
public void start() { public void start() {
createExecutors(); createExecutors();
startFeedProcessingLoop(); startFeedProcessingLoop();
startRefillLoop(); startRefillLoop();
} }
private void startFeedProcessingLoop() { private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat // take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> { feedProcessingLoopExecutor.submit(
while (!feedProcessingLoopExecutor.isShutdown()) { () -> {
try { while (!feedProcessingLoopExecutor.isShutdown()) {
// take() is blocking until a feed is available from the queue try {
Feed feed = queue.take(); // take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed // send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId()); log.debug(
processFeedAsync(feed); "got feed {} from the queue, send it for processing",
feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty // we removed a feed from the queue, try to refill it as it may now be
if (queue.isEmpty()) { // empty
log.debug("took the last feed from the queue, try to refill"); if (queue.isEmpty()) {
refillQueueAsync(); log.debug("took the last feed from the queue, try to refill");
} refillQueueAsync();
} catch (InterruptedException e) { }
log.debug("interrupted while waiting for a feed in the queue"); } catch (InterruptedException e) {
Thread.currentThread().interrupt(); log.debug("interrupted while waiting for a feed in the queue");
} catch (Exception e) { Thread.currentThread().interrupt();
log.error(e.getMessage(), e); } catch (Exception e) {
} log.error(e.getMessage(), e);
} }
}); }
} });
}
private void startRefillLoop() { private void startRefillLoop() {
// refill the queue at regular intervals if it's empty // refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> { refillLoopExecutor.submit(
while (!refillLoopExecutor.isShutdown()) { () -> {
try { while (!refillLoopExecutor.isShutdown()) {
if (queue.isEmpty()) { try {
log.debug("refilling queue"); if (queue.isEmpty()) {
refillQueueAsync(); log.debug("refilling queue");
} refillQueueAsync();
}
log.debug("sleeping for 15s"); log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15); TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.debug("interrupted while sleeping"); log.debug("interrupted while sleeping");
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} catch (Exception e) { } catch (Exception e) {
log.error(e.getMessage(), e); log.error(e.getMessage(), e);
} }
} }
}); });
} }
public void refreshImmediately(Feed feed) { public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId()); log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice // remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId())); queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed); queue.addFirst(feed);
} }
private void refillQueueAsync() { private void refillQueueAsync() {
CompletableFuture.runAsync(() -> { CompletableFuture.runAsync(
if (!queue.isEmpty()) { () -> {
return; if (!queue.isEmpty()) {
} return;
}
refill.mark(); refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize()); List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size()); log.debug(
for (Feed feed : nextUpdatableFeeds) { "found {} feeds that are up for refresh",
// add the feed only if it was not already queued nextUpdatableFeeds.size());
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) { for (Feed feed : nextUpdatableFeeds) {
queue.addLast(feed); // add the feed only if it was not already queued
} if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
} queue.addLast(feed);
}, refillExecutor).whenComplete((data, ex) -> { }
if (ex != null) { }
log.error("error while refilling the queue", ex); },
} refillExecutor)
}); .whenComplete(
} (data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) { private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor) CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor) .thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.thenCompose(r -> { .thenCompose(
List<CompletableFuture<Void>> futures = r.insertedUnreadEntriesBySubscription().entrySet().stream().map(e -> { r -> {
FeedSubscription sub = e.getKey(); List<CompletableFuture<Void>> futures =
List<FeedEntry> entries = e.getValue(); r.insertedUnreadEntriesBySubscription().entrySet().stream()
.map(
e -> {
FeedSubscription sub = e.getKey();
List<FeedEntry> entries = e.getValue();
notifier.notifyOverWebsocket(sub, entries); notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(() -> notifier.sendPushNotifications(sub, entries), notifierExecutor); return CompletableFuture.runAsync(
}).toList(); () ->
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)); notifier
}) .sendPushNotifications(
.exceptionally(ex -> { sub,
log.error("error while processing feed {}", feed.getUrl(), ex); entries),
return null; notifierExecutor);
}); })
} .toList();
return CompletableFuture.allOf(
futures.toArray(CompletableFuture[]::new));
})
.exceptionally(
ex -> {
log.error("error while processing feed {}", feed.getUrl(), ex);
return null;
});
}
private List<Feed> getNextUpdatableFeeds(int max) { private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> { return unitOfWork.call(
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null () -> {
: Instant.now().minus(config.feedRefresh().userInactivityPeriod()); Instant lastLoginThreshold =
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold); config.feedRefresh().userInactivityPeriod().isZero()
if (!feeds.isEmpty()) { ? null
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable() : Instant.now()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval()); .minus(config.feedRefresh().userInactivityPeriod());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate); List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
} if (!feeds.isEmpty()) {
return feeds; // update disabledUntil to prevent feeds from being returned again by
}); // feedDAO.findNextUpdatable()
} Instant nextUpdateDate =
Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(
feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
}
return feeds;
});
}
private int getBatchSize() { private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads()); return Math.min(100, 3 * config.feedRefresh().httpThreads());
} }
public void stop() { public void stop() {
MoreExecutors.shutdownAndAwaitTermination(this.feedProcessingLoopExecutor, config.shutdownTimeout()); MoreExecutors.shutdownAndAwaitTermination(
MoreExecutors.shutdownAndAwaitTermination(this.refillLoopExecutor, config.shutdownTimeout()); this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout()); MoreExecutors.shutdownAndAwaitTermination(
MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout()); this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.databaseUpdaterExecutor, config.shutdownTimeout()); MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout()); MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
queue.clear(); queue.clear();
} }
/** /**
* returns an ExecutorService with a single thread that discards tasks if a task is already running * returns an ExecutorService with a single thread that discards tasks if a task is already
*/ * running
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() { */
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>()); private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); ThreadPoolExecutor pool =
return pool; new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
} pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/** /** returns an ExecutorService that discards tasks if the queue is full */
* returns an ExecutorService that discards tasks if the queue is full private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
*/ ThreadPoolExecutor pool =
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) { new ThreadPoolExecutor(
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, threads,
new LinkedBlockingQueue<>(queueCapacity)); threads,
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); 0L,
return pool; TimeUnit.MILLISECONDS,
} new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/** /** returns an ExecutorService that blocks submissions until a thread is available */
* returns an ExecutorService that blocks submissions until a thread is available private ThreadPoolExecutor newBlockingExecutorService(int threads) {
*/ ThreadPoolExecutor pool =
private ThreadPoolExecutor newBlockingExecutorService(int threads) { new ThreadPoolExecutor(
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>()); threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> { pool.setRejectedExecutionHandler(
if (e.isShutdown()) { (r, e) -> {
return; if (e.isShutdown()) {
} return;
}
try { try {
e.getQueue().put(r); e.getQueue().put(r);
} catch (InterruptedException ex) { } catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex); log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
}); });
return pool; return pool;
} }
} }

139
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -1,84 +1,95 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling; import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs; import com.google.common.primitives.Longs;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import org.apache.commons.lang3.ObjectUtils;
@Singleton @Singleton
public class FeedRefreshIntervalCalculator { public class FeedRefreshIntervalCalculator {
private final Duration interval; private final Duration interval;
private final Duration maxInterval; private final Duration maxInterval;
private final boolean empirical; private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling; private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource; private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) { public FeedRefreshIntervalCalculator(
this.interval = config.feedRefresh().interval(); CommaFeedConfiguration config, InstantSource instantSource) {
this.maxInterval = config.feedRefresh().maxInterval(); this.interval = config.feedRefresh().interval();
this.empirical = config.feedRefresh().intervalEmpirical(); this.maxInterval = config.feedRefresh().maxInterval();
this.errorHandling = config.feedRefresh().errors(); this.empirical = config.feedRefresh().intervalEmpirical();
this.instantSource = instantSource; this.errorHandling = config.feedRefresh().errors();
} this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) { public Instant onFetchSuccess(
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval) Instant publishedDate, Long averageEntryInterval, Duration validFor) {
: instantSource.instant().plus(interval); Instant instant =
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor))); empirical
} ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) { public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO); return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
} }
public Instant onTooManyRequests(Instant retryAfter, int errorCount) { public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount))); return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
} }
public Instant onFetchError(int errorCount) { public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) { if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval)); return constrainToBounds(instantSource.instant().plus(interval));
} }
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L); Duration retryInterval =
return constrainToBounds(instantSource.instant().plus(retryInterval)); errorHandling
} .backoffInterval()
.multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) { private Instant computeEmpiricalRefreshInterval(
Instant now = instantSource.instant(); Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) { if (publishedDate == null) {
return now.plus(maxInterval); return now.plus(maxInterval);
} }
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now); long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) { if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval); return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) { } else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2)); return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) { } else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4)); return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) { } else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor // use average time between entries to decide when to refresh next, divided by factor
int factor = 2; int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis()); long millis =
return now.plusMillis(millis); Longs.constrainToRange(
} else { averageEntryInterval / factor,
// unknown case interval.toMillis(),
return now.plus(maxInterval); maxInterval.dividedBy(4).toMillis());
} return now.plusMillis(millis);
} } else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) { private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval)); return ObjectUtils.max(
} ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)),
instantSource.instant().plus(interval));
}
} }

270
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -1,20 +1,5 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests; import com.commafeed.backend.Digests;
@@ -29,142 +14,171 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService; import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService; import com.commafeed.backend.service.FeedService;
import com.google.common.util.concurrent.Striped; import com.google.common.util.concurrent.Striped;
import jakarta.inject.Singleton;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
/** /** Updates the feed in the database and inserts new entries */
* Updates the feed in the database and inserts new entries
*/
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedRefreshUpdater { public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final FeedService feedService; private final FeedService feedService;
private final FeedEntryService feedEntryService; private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO; private final FeedSubscriptionDAO feedSubscriptionDAO;
private final Striped<Lock> locks; private final Striped<Lock> locks;
private final Meter feedUpdated; private final Meter feedUpdated;
private final Meter entryInserted; private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics, public FeedRefreshUpdater(
FeedSubscriptionDAO feedSubscriptionDAO) { UnitOfWork unitOfWork,
this.unitOfWork = unitOfWork; FeedService feedService,
this.feedService = feedService; FeedEntryService feedEntryService,
this.feedEntryService = feedEntryService; MetricRegistry metrics,
this.feedSubscriptionDAO = feedSubscriptionDAO; FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
locks = Striped.lazyWeakLock(100000); locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated")); feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted")); entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
} }
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) { private AddEntryResult addEntry(
boolean processed = false; final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
FeedEntry insertedEntry = null; boolean processed = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>(); FeedEntry insertedEntry = null;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at // lock on feed, make sure we are not updating the same feed twice at
// the same time // the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId())); String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry // lock on content, make sure we are not updating the same entry
// twice at the same time // twice at the same time
Content content = entry.content(); Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title())); String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator(); Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next(); Lock lock1 = iterator.next();
Lock lock2 = iterator.next(); Lock lock2 = iterator.next();
boolean locked1 = false; boolean locked1 = false;
boolean locked2 = false; boolean locked2 = false;
try { try {
// try to lock, give up after 1 minute // try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES); locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES); locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) { if (locked1 && locked2) {
processed = true; processed = true;
insertedEntry = unitOfWork.call(() -> { insertedEntry =
if (feedEntryService.find(feed, entry) != null) { unitOfWork.call(
// entry already exists, nothing to do () -> {
return null; if (feedEntryService.find(feed, entry) != null) {
} // entry already exists, nothing to do
return null;
}
FeedEntry feedEntry = feedEntryService.create(feed, entry); FeedEntry feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark(); entryInserted.mark();
for (FeedSubscription sub : subscriptions) { for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry); boolean unread =
if (unread) { feedEntryService.applyFilter(sub, feedEntry);
subscriptionsForWhichEntryIsUnread.add(sub); if (unread) {
} subscriptionsForWhichEntryIsUnread.add(sub);
} }
return feedEntry; }
}); return feedEntry;
} else { });
log.error("lock timeout for {} - {}", feed.getUrl(), key1); } else {
} log.error("lock timeout for {} - {}", feed.getUrl(), key1);
} catch (InterruptedException e) { }
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e); } catch (InterruptedException e) {
Thread.currentThread().interrupt(); log.error(
} finally { "interrupted while waiting for lock for {} : {}",
if (locked1) { feed.getUrl(),
lock1.unlock(); e.getMessage(),
} e);
if (locked2) { Thread.currentThread().interrupt();
lock2.unlock(); } finally {
} if (locked1) {
} lock1.unlock();
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread); }
} if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
}
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) { public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true; boolean processed = true;
long inserted = 0; long inserted = 0;
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>(); Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription =
new HashMap<>();
if (!entries.isEmpty()) { if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null; List<FeedSubscription> subscriptions = null;
List<Entry> newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries)); List<Entry> newEntries =
for (Entry entry : newEntries) { unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
if (subscriptions == null) { for (Entry entry : newEntries) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed)); if (subscriptions == null) {
} subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions); }
processed &= addEntryResult.processed; AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
inserted += addEntryResult.insertedEntry != null ? 1 : 0; processed &= addEntryResult.processed;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> { inserted += addEntryResult.insertedEntry != null ? 1 : 0;
if (addEntryResult.insertedEntry != null) { addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(
insertedUnreadEntriesBySubscription.computeIfAbsent(sub, k -> new ArrayList<>()).add(addEntryResult.insertedEntry); sub -> {
} if (addEntryResult.insertedEntry != null) {
}); insertedUnreadEntriesBySubscription
} .computeIfAbsent(sub, k -> new ArrayList<>())
.add(addEntryResult.insertedEntry);
}
});
}
if (inserted == 0) { if (inserted == 0) {
feed.setMessage("No new entries found"); feed.setMessage("No new entries found");
} else if (inserted > 0) { } else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted)); feed.setMessage("Found %s new entries".formatted(inserted));
} }
} }
if (!processed) { if (!processed) {
// requeue asap // requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT); feed.setDisabledUntil(Models.MINIMUM_INSTANT);
} }
if (inserted > 0) { if (inserted > 0) {
feedUpdated.mark(); feedUpdated.mark();
} }
unitOfWork.run(() -> feedService.update(feed)); unitOfWork.run(() -> feedService.update(feed));
return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription); return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
} }
private record AddEntryResult(boolean processed, FeedEntry insertedEntry, Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {} private record AddEntryResult(
boolean processed,
public record FeedRefreshUpdaterResult(Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {} FeedEntry insertedEntry,
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
public record FeedRefreshUpdaterResult(
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {}
} }

190
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,15 +1,5 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.Strings;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
@@ -18,108 +8,132 @@ import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult; import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry; import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Strings;
/** /**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that) * Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link
* FeedRefreshUpdater} does that)
*/ */
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedRefreshWorker { public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator; private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher; private final FeedFetcher fetcher;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final Meter feedFetched; private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config, public FeedRefreshWorker(
MetricRegistry metrics) { FeedRefreshIntervalCalculator refreshIntervalCalculator,
this.refreshIntervalCalculator = refreshIntervalCalculator; FeedFetcher fetcher,
this.fetcher = fetcher; CommaFeedConfiguration config,
this.config = config; MetricRegistry metrics) {
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched")); this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
} public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result =
fetcher.fetch(
url,
false,
feed.getLastModifiedHeader(),
feed.getEtagHeader(),
feed.getLastPublishedDate(),
feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
public FeedRefreshWorkerResult update(Feed feed) { List<Entry> entries = result.feed().entries();
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries(); int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity(); Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (maxFeedCapacity > 0) { if (!entriesMaxAge.isZero()) {
entries = entries.stream().limit(maxFeedCapacity).toList(); Instant threshold = Instant.now().minus(entriesMaxAge);
} entries =
entries.stream()
.filter(entry -> entry.published().isAfter(threshold))
.toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge(); String urlAfterRedirect = result.urlAfterRedirect();
if (!entriesMaxAge.isZero()) { if (Strings.CS.equals(url, urlAfterRedirect)) {
Instant threshold = Instant.now().minus(entriesMaxAge); urlAfterRedirect = null;
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList(); }
}
String urlAfterRedirect = result.urlAfterRedirect(); feed.setUrlAfterRedirect(urlAfterRedirect);
if (Strings.CS.equals(url, urlAfterRedirect)) { feed.setLink(result.feed().link());
urlAfterRedirect = null; feed.setIconUrl(result.feed().iconUrl());
} feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setUrlAfterRedirect(urlAfterRedirect); feed.setErrorCount(0);
feed.setLink(result.feed().link()); feed.setMessage(null);
feed.setIconUrl(result.feed().iconUrl()); feed.setDisabledUntil(
feed.setLastModifiedHeader(result.lastModifiedHeader()); refreshIntervalCalculator.onFetchSuccess(
feed.setEtagHeader(result.lastETagHeader()); result.feed().lastPublishedDate(),
feed.setLastContentHash(result.contentHash()); result.feed().averageEntryInterval(),
feed.setLastPublishedDate(result.feed().lastPublishedDate()); result.validFor()));
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0); return new FeedRefreshWorkerResult(feed, entries);
feed.setMessage(null); } catch (NotModifiedException e) {
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries); feed.setErrorCount(0);
} catch (NotModifiedException e) { feed.setMessage(e.getMessage());
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage()); feed.setDisabledUntil(
refreshIntervalCalculator.onFeedNotModified(
feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
feed.setErrorCount(0); if (e.getNewLastModifiedHeader() != null) {
feed.setMessage(e.getMessage()); feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval())); }
if (e.getNewLastModifiedHeader() != null) { if (e.getNewEtagHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader()); feed.setEtagHeader(e.getNewEtagHeader());
} }
if (e.getNewEtagHeader() != null) { return new FeedRefreshWorkerResult(feed, Collections.emptyList());
feed.setEtagHeader(e.getNewEtagHeader()); } catch (TooManyRequestsException e) {
} log.debug("Too many requests : {}", feed.getUrl());
return new FeedRefreshWorkerResult(feed, Collections.emptyList()); feed.setErrorCount(feed.getErrorCount() + 1);
} catch (TooManyRequestsException e) { feed.setMessage("Server indicated that we are sending too many requests");
log.debug("Too many requests : {}", feed.getUrl()); feed.setDisabledUntil(
refreshIntervalCalculator.onTooManyRequests(
e.getRetryAfter(), feed.getErrorCount()));
feed.setErrorCount(feed.getErrorCount() + 1); return new FeedRefreshWorkerResult(feed, Collections.emptyList());
feed.setMessage("Server indicated that we are sending too many requests"); } catch (Exception e) {
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount())); log.debug("unable to refresh feed {}", feed.getUrl(), e);
return new FeedRefreshWorkerResult(feed, Collections.emptyList()); feed.setErrorCount(feed.getErrorCount() + 1);
} catch (Exception e) { feed.setMessage("Unable to refresh feed : " + e.getMessage());
log.debug("unable to refresh feed {}", feed.getUrl(), e); feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
feed.setErrorCount(feed.getErrorCount() + 1); return new FeedRefreshWorkerResult(feed, Collections.emptyList());
feed.setMessage("Unable to refresh feed : " + e.getMessage()); } finally {
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount())); feedFetched.mark();
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList()); }
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {}
} }

55
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java
View File

@@ -1,9 +1,5 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserSettingsDAO; import com.commafeed.backend.dao.UserSettingsDAO;
@@ -13,7 +9,8 @@ import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.PushNotificationService; import com.commafeed.backend.service.PushNotificationService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder; import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions; import com.commafeed.frontend.ws.WebSocketSessions;
import jakarta.inject.Singleton;
import java.util.List;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -22,29 +19,33 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor @RequiredArgsConstructor
public class FeedUpdateNotifier { public class FeedUpdateNotifier {
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final UserSettingsDAO userSettingsDAO; private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions; private final WebSocketSessions webSocketSessions;
private final PushNotificationService pushNotificationService; private final PushNotificationService pushNotificationService;
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) { public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) { if (!entries.isEmpty()) {
webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size())); webSocketSessions.sendMessage(
} sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
} }
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) { public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled() || !sub.isPushNotificationsEnabled() || entries.isEmpty()) { if (!config.pushNotifications().enabled()
return; || !sub.isPushNotificationsEnabled()
} || entries.isEmpty()) {
return;
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser())); }
if (settings != null && settings.getPushNotifications() != null && settings.getPushNotifications().getType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
}
}
}
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null
&& settings.getPushNotifications() != null
&& settings.getPushNotifications().getType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
}
}
}
} }

136
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,14 +1,5 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.Date;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.commafeed.backend.feed.parser.TextDirectionDetector; import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry; import com.commafeed.frontend.model.Entry;
@@ -16,87 +7,90 @@ import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl; import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry; import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl; import com.rometools.rome.feed.synd.SyndEntryImpl;
import java.util.Collections;
import java.util.Date;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/** /** Utility methods related to feed handling */
* Utility methods related to feed handling
*
*/
@UtilityClass @UtilityClass
@Slf4j @Slf4j
public class FeedUtils { public class FeedUtils {
public static String truncate(String string, int length) { public static String truncate(String string, int length) {
return StringUtils.truncate(string, length); return StringUtils.truncate(string, length);
} }
public static boolean isRTL(String title, String content) { public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title; String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) { if (StringUtils.isBlank(text)) {
return false; return false;
} }
String stripped = Jsoup.parse(text).text(); String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) { if (StringUtils.isBlank(stripped)) {
return false; return false;
} }
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT; return TextDirectionDetector.detect(stripped)
} == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String getFaviconUrl(FeedSubscription subscription) { public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId(); return "rest/feed/favicon/" + subscription.getId();
} }
public static String proxyImages(String content) { public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) { if (StringUtils.isBlank(content)) {
return content; return content;
} }
Document doc = Jsoup.parse(content); Document doc = Jsoup.parse(content);
Elements elements = doc.select("img"); Elements elements = doc.select("img");
for (Element element : elements) { for (Element element : elements) {
String href = element.attr("src"); String href = element.attr("src");
if (StringUtils.isNotBlank(href)) { if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href); String proxy = proxyImage(href);
element.attr("src", proxy); element.attr("src", proxy);
} }
} }
return doc.body().html(); return doc.body().html();
} }
public static String proxyImage(String url) { public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) { if (StringUtils.isBlank(url)) {
return url; return url;
} }
return "rest/server/proxy?u=" + ImageProxyUrl.encode(url); return "rest/server/proxy?u=" + ImageProxyUrl.encode(url);
} }
public static SyndEntry asRss(Entry entry) { public static SyndEntry asRss(Entry entry) {
SyndEntry e = new SyndEntryImpl(); SyndEntry e = new SyndEntryImpl();
e.setUri(entry.getGuid()); e.setUri(entry.getGuid());
e.setTitle(entry.getTitle()); e.setTitle(entry.getTitle());
e.setAuthor(entry.getAuthor()); e.setAuthor(entry.getAuthor());
SyndContentImpl c = new SyndContentImpl(); SyndContentImpl c = new SyndContentImpl();
c.setValue(entry.getContent()); c.setValue(entry.getContent());
e.setContents(Collections.singletonList(c)); e.setContents(Collections.singletonList(c));
if (entry.getEnclosureUrl() != null) { if (entry.getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl(); SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(entry.getEnclosureType()); enclosure.setType(entry.getEnclosureType());
enclosure.setUrl(entry.getEnclosureUrl()); enclosure.setUrl(entry.getEnclosureUrl());
e.setEnclosures(Collections.singletonList(enclosure)); e.setEnclosures(Collections.singletonList(enclosure));
} }
e.setLink(entry.getUrl());
e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
return e;
}
e.setLink(entry.getUrl());
e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
return e;
}
} }

83
commafeed-server/src/main/java/com/commafeed/backend/feed/ImageProxyUrl.java
View File

@@ -1,70 +1,65 @@
package com.commafeed.backend.feed; package com.commafeed.backend.feed;
import com.google.common.primitives.Bytes;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Arrays; import java.util.Arrays;
import java.util.Base64; import java.util.Base64;
import javax.crypto.Cipher; import javax.crypto.Cipher;
import javax.crypto.SecretKey; import javax.crypto.SecretKey;
import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec; import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.lang3.RandomUtils;
import com.google.common.primitives.Bytes;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import org.apache.commons.lang3.RandomUtils;
@UtilityClass @UtilityClass
public class ImageProxyUrl { public class ImageProxyUrl {
private static final int GCM_IV_LENGTH = 12; private static final int GCM_IV_LENGTH = 12;
private static final int GCM_TAG_LENGTH = 128; private static final int GCM_TAG_LENGTH = 128;
private static SecretKey key; private static SecretKey key;
public static void generateKey() { public static void generateKey() {
key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES"); key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES");
} }
public static String encode(String url) { public static String encode(String url) {
if (key == null) { if (key == null) {
throw new IllegalStateException("Key not initialized"); throw new IllegalStateException("Key not initialized");
} }
try { try {
byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH); byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv)); cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8)); byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8));
byte[] combined = Bytes.concat(iv, encrypted); byte[] combined = Bytes.concat(iv, encrypted);
return Base64.getUrlEncoder().withoutPadding().encodeToString(combined); return Base64.getUrlEncoder().withoutPadding().encodeToString(combined);
} catch (Exception e) { } catch (Exception e) {
throw new IllegalStateException("Failed to encode URL", e); throw new IllegalStateException("Failed to encode URL", e);
} }
} }
public static String decode(String code) { public static String decode(String code) {
if (key == null) { if (key == null) {
throw new IllegalStateException("Key not initialized"); throw new IllegalStateException("Key not initialized");
} }
try { try {
byte[] combined = Base64.getUrlDecoder().decode(code); byte[] combined = Base64.getUrlDecoder().decode(code);
byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH); byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH);
byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length); byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv)); cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] decrypted = cipher.doFinal(encrypted); byte[] decrypted = cipher.doFinal(encrypted);
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new IllegalStateException("Failed to decode URL", e);
}
}
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new IllegalStateException("Failed to decode URL", e);
}
}
} }

105
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -1,70 +1,61 @@
package com.commafeed.backend.feed.parser; package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.Strings;
import com.ibm.icu.text.CharsetDetector; import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch; import com.ibm.icu.text.CharsetMatch;
import jakarta.inject.Singleton;
import java.nio.charset.Charset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.Strings;
@Singleton @Singleton
public class EncodingDetector { public class EncodingDetector {
/** /**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the * Detect feed encoding by using the declared encoding in the xml processing instruction and by
* feed * detecting the characters used in the feed
* */
*/ public Charset getEncoding(byte[] bytes) {
public Charset getEncoding(byte[] bytes) { String extracted = extractDeclaredEncoding(bytes);
String extracted = extractDeclaredEncoding(bytes); if (Strings.CI.startsWith(extracted, "iso-8859-")) {
if (Strings.CI.startsWith(extracted, "iso-8859-")) { if (!Strings.CS.endsWith(extracted, "1")) {
if (!Strings.CS.endsWith(extracted, "1")) { return Charset.forName(extracted);
return Charset.forName(extracted); }
} } else if (Strings.CI.startsWith(extracted, "windows-")) {
} else if (Strings.CI.startsWith(extracted, "windows-")) { return Charset.forName(extracted);
return Charset.forName(extracted); }
} return detectEncoding(bytes);
return detectEncoding(bytes); }
}
/** /** Extract the declared encoding from the xml */
* Extract the declared encoding from the xml public String extractDeclaredEncoding(byte[] bytes) {
*/ int index = ArrayUtils.indexOf(bytes, (byte) '>');
public String extractDeclaredEncoding(byte[] bytes) { if (index == -1) {
int index = ArrayUtils.indexOf(bytes, (byte) '>'); return null;
if (index == -1) { }
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"'); String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = Strings.CS.indexOf(pi, "encoding=\""); index = Strings.CS.indexOf(pi, "encoding=\"");
if (index == -1) { if (index == -1) {
return null; return null;
} }
String encoding = pi.substring(index + 10); String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"')); encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding; return encoding;
} }
/** /** Detect encoding by analyzing characters in the array */
* Detect encoding by analyzing characters in the array private Charset detectEncoding(byte[] bytes) {
*/ String encoding = "UTF-8";
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
} }

468
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -1,25 +1,5 @@
package com.commafeed.backend.feed.parser; package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemProperties;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.Urls; import com.commafeed.backend.Urls;
import com.commafeed.backend.feed.parser.FeedParserResult.Content; import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure; import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
@@ -38,258 +18,290 @@ import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink; import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl; import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput; import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemProperties;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
/** /** Parses raw xml into a FeedParserResult object */
* Parses raw xml into a FeedParserResult object
*/
@Singleton @Singleton
public class FeedParser { public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom"); private static final Namespace ATOM_10_NS =
Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000); private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000); private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private static final Comparator<Entry> ENTRY_COMPARATOR = Comparator.comparing(Entry::published).reversed(); private static final Comparator<Entry> ENTRY_COMPARATOR =
Comparator.comparing(Entry::published).reversed();
private final EncodingDetector encodingDetector; private final EncodingDetector encodingDetector;
private final XMLCleaner xmlCleaner; private final XMLCleaner xmlCleaner;
public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) { public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) {
this.encodingDetector = encodingDetector; this.encodingDetector = encodingDetector;
this.xmlCleaner = xmlCleaner; this.xmlCleaner = xmlCleaner;
// disable entity expansion limits added in JDK24+ (#1961) // disable entity expansion limits added in JDK24+ (#1961)
// we already strip doctype declarations in XMLCleaner to prevent xxe attacks // we already strip doctype declarations in XMLCleaner to prevent xxe attacks
// we also already limit the size of feeds we download in HttpGetter // we also already limit the size of feeds we download in HttpGetter
System.setProperty(SystemProperties.JDK_XML_MAX_GENERAL_ENTITY_SIZE_LIMIT, "0"); System.setProperty(SystemProperties.JDK_XML_MAX_GENERAL_ENTITY_SIZE_LIMIT, "0");
System.setProperty(SystemProperties.JDK_XML_TOTAL_ENTITY_SIZE_LIMIT, "0"); System.setProperty(SystemProperties.JDK_XML_TOTAL_ENTITY_SIZE_LIMIT, "0");
} }
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException { public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try { try {
Charset encoding = encodingDetector.getEncoding(xml); Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = xmlCleaner.clean(new String(xml, encoding)); String xmlString = xmlCleaner.clean(new String(xml, encoding));
if (xmlString == null) { if (xmlString == null) {
throw new FeedParsingException("Input string is empty for url " + feedUrl); throw new FeedParsingException("Input string is empty for url " + feedUrl);
} }
InputSource source = new InputSource(new StringReader(xmlString)); InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source); SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed); handleForeignMarkup(feed);
String title = feed.getTitle(); String title = feed.getTitle();
String link = feed.getLink(); String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null; String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl); List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null); Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false); Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) { if (lastPublishedDate == null
lastPublishedDate = lastEntryDate; || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
} lastPublishedDate = lastEntryDate;
Long averageEntryInterval = averageTimeBetweenEntries(entries); }
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries); return new FeedParserResult(
} catch (FeedParsingException e) { title,
throw e; link,
} catch (Exception e) { iconUrl,
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e); lastPublishedDate,
} averageEntryInterval,
} lastEntryDate,
entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(
String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/** /** Adds atom links for rss feeds */
* Adds atom links for rss feeds private void handleForeignMarkup(SyndFeed feed) {
*/ List<Element> foreignMarkup = feed.getForeignMarkup();
private void handleForeignMarkup(SyndFeed feed) { if (foreignMarkup == null) {
List<Element> foreignMarkup = feed.getForeignMarkup(); return;
if (foreignMarkup == null) { }
return; for (Element element : foreignMarkup) {
} if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
for (Element element : foreignMarkup) { SyndLink link = new SyndLinkImpl();
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) { link.setRel(element.getAttributeValue("rel"));
SyndLink link = new SyndLinkImpl(); link.setHref(element.getAttributeValue("href"));
link.setRel(element.getAttributeValue("rel")); feed.getLinks().add(link);
link.setHref(element.getAttributeValue("href")); }
feed.getLinks().add(link); }
} }
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) { private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>(); List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) { for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri(); String guid = item.getUri();
if (StringUtils.isBlank(guid)) { if (StringUtils.isBlank(guid)) {
guid = item.getLink(); guid = item.getLink();
} }
if (StringUtils.isBlank(guid)) { if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry // no guid and no link, skip entry
continue; continue;
} }
String url = buildEntryUrl(feed, feedUrl, item); String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) { if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
// if link is empty but guid is used as url, use guid // if link is empty but guid is used as url, use guid
url = guid; url = guid;
} }
Instant publishedDate = buildEntryPublishedDate(item); Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item); Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content)); entries.add(new Entry(guid, url, publishedDate, content));
} }
entries.sort(ENTRY_COMPARATOR); entries.sort(ENTRY_COMPARATOR);
return entries; return entries;
} }
private Content buildContent(SyndEntry item) { private Content buildContent(SyndEntry item) {
String title = getTitle(item); String title = getTitle(item);
String content = getContent(item); String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor()); String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils String categories =
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", "))); StringUtils.trimToNull(
item.getCategories().stream()
.map(SyndCategory::getName)
.collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item); Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item); Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media); return new Content(title, content, author, categories, enclosure, media);
} }
private Enclosure buildEnclosure(SyndEntry item) { private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null); SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null);
if (enclosure == null) { if (enclosure == null) {
return null; return null;
} }
return new Enclosure(enclosure.getUrl(), enclosure.getType()); return new Enclosure(enclosure.getUrl(), enclosure.getType());
} }
private Instant buildEntryPublishedDate(SyndEntry item) { private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate(); Date date = item.getPublishedDate();
if (date == null) { if (date == null) {
date = item.getUpdatedDate(); date = item.getUpdatedDate();
} }
return toValidInstant(date, true); return toValidInstant(date, true);
} }
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) { private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink())); String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || Urls.isAbsolute(url)) { if (url == null || Urls.isAbsolute(url)) {
// url is absolute, nothing to do // url is absolute, nothing to do
return url; return url;
} }
// url is relative, trying to resolve it // url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink())); String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return Urls.toAbsolute(url, feedLink, feedUrl); return Urls.toAbsolute(url, feedLink, feedUrl);
} }
private Instant toValidInstant(Date date, boolean nullToNow) { private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now(); Instant now = Instant.now();
if (date == null) { if (date == null) {
return nullToNow ? now : null; return nullToNow ? now : null;
} }
Instant instant = date.toInstant(); Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) { if (instant.isBefore(START) || instant.isAfter(END)) {
return now; return now;
} }
if (instant.isAfter(now)) { if (instant.isAfter(now)) {
return now; return now;
} }
return instant; return instant;
} }
private String getContent(SyndEntry item) { private String getContent(SyndEntry item) {
String content; String content;
if (item.getContents().isEmpty()) { if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue(); content = item.getDescription() == null ? null : item.getDescription().getValue();
} else { } else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator())); content =
} item.getContents().stream()
return StringUtils.trimToNull(content); .map(SyndContent::getValue)
} .collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) { private String getTitle(SyndEntry item) {
String title = item.getTitle(); String title = item.getTitle();
if (StringUtils.isBlank(title)) { if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate(); Date date = item.getPublishedDate();
if (date != null) { if (date != null) {
title = DateFormat.getInstance().format(date); title = DateFormat.getInstance().format(date);
} else { } else {
title = "(no title)"; title = "(no title)";
} }
} }
return StringUtils.trimToNull(title); return StringUtils.trimToNull(title);
} }
private Media buildMedia(SyndEntry item) { private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI); MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) { if (module == null) {
return null; return null;
} }
Media media = buildMedia(module.getMetadata()); Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) { if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0]; MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata()); media = buildMedia(group.getMetadata());
} }
return media; return media;
} }
private Media buildMedia(Metadata metadata) { private Media buildMedia(Metadata metadata) {
if (metadata == null) { if (metadata == null) {
return null; return null;
} }
String description = metadata.getDescription(); String description = metadata.getDescription();
String thumbnailUrl = null; String thumbnailUrl = null;
Integer thumbnailWidth = null; Integer thumbnailWidth = null;
Integer thumbnailHeight = null; Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) { if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0]; Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth(); thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight(); thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) { if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString(); thumbnailUrl = thumbnail.getUrl().toString();
} }
} }
if (description == null && thumbnailUrl == null) { if (description == null && thumbnailUrl == null) {
return null; return null;
} }
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight); return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
} }
private Long averageTimeBetweenEntries(List<Entry> entries) { private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) { if (entries.isEmpty() || entries.size() == 1) {
return null; return null;
} }
SummaryStatistics stats = new SummaryStatistics(); SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) { for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli()); long diff =
stats.addValue(diff); Math.abs(
} entries.get(i).published().toEpochMilli()
return (long) stats.getMean(); - entries.get(i + 1).published().toEpochMilli());
} stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception { public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public FeedParsingException(String message) { public FeedParsingException(String message) {
super(message); super(message);
} }
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
} }

28
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -3,13 +3,29 @@ package com.commafeed.backend.feed.parser;
import java.time.Instant; import java.time.Instant;
import java.util.List; import java.util.List;
public record FeedParserResult(String title, String link, String iconUrl, Instant lastPublishedDate, Long averageEntryInterval, public record FeedParserResult(
Instant lastEntryDate, List<Entry> entries) { String title,
public record Entry(String guid, String url, Instant published, Content content) {} String link,
String iconUrl,
Instant lastPublishedDate,
Long averageEntryInterval,
Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {} public record Content(
String title,
String content,
String author,
String categories,
Enclosure enclosure,
Media media) {}
public record Enclosure(String url, String type) {} public record Enclosure(String url, String type) {}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {} public record Media(
String description,
String thumbnailUrl,
Integer thumbnailWidth,
Integer thumbnailHeight) {}
} }

515
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
View File

@@ -4,268 +4,267 @@ import java.util.Collections;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
class HtmlEntities { class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP; public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final List<String> HTML_ENTITIES; public static final List<String> HTML_ENTITIES;
static { static {
Map<String, String> map = new LinkedHashMap<>(); Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;"); map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;"); map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;"); map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;"); map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;"); map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;"); map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;"); map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;"); map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;"); map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;"); map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;"); map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;"); map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;"); map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;"); map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;"); map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;"); map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;"); map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;"); map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;"); map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;"); map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;"); map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;"); map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;"); map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;"); map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;"); map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;"); map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;"); map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;"); map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;"); map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;"); map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;"); map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;"); map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;"); map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;"); map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;"); map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;"); map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;"); map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;"); map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;"); map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;"); map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;"); map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;"); map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;"); map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;"); map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;"); map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;"); map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;"); map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;"); map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;"); map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;"); map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;"); map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;"); map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;"); map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;"); map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;"); map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;"); map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;"); map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;"); map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;"); map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;"); map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;"); map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;"); map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;"); map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;"); map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;"); map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;"); map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;"); map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;"); map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;"); map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;"); map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;"); map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;"); map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;"); map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;"); map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;"); map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;"); map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;"); map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;"); map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;"); map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;"); map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;"); map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;"); map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;"); map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;"); map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;"); map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;"); map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;"); map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;"); map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;"); map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;"); map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;"); map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;"); map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;"); map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;"); map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;"); map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;"); map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;"); map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;"); map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;"); map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;"); map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;"); map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;"); map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;"); map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;"); map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;"); map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;"); map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;"); map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;"); map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;"); map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;"); map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;"); map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;"); map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;"); map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;"); map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;"); map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;"); map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;"); map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;"); map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;"); map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;"); map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;"); map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;"); map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;"); map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;"); map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;"); map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;"); map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;"); map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;"); map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;"); map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;"); map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;"); map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;"); map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;"); map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;"); map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;"); map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;"); map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;"); map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;"); map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;"); map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;"); map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;"); map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;"); map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;"); map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;"); map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;"); map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;"); map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;"); map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;"); map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;"); map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;"); map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;"); map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;"); map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;"); map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;"); map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;"); map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;"); map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;"); map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;"); map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;"); map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;"); map.put("&para;", "&#182;");
map.put("&part;", "&#8706;"); map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;"); map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;"); map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;"); map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;"); map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;"); map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;"); map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;"); map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;"); map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;"); map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;"); map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;"); map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;"); map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;"); map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;"); map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;"); map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;"); map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;"); map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;"); map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;"); map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;"); map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;"); map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;"); map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;"); map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;"); map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;"); map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;"); map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;"); map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;"); map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;"); map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;"); map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;"); map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;"); map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;"); map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;"); map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;"); map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;"); map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;"); map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;"); map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;"); map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;"); map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;"); map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;"); map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;"); map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;"); map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;"); map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;"); map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;"); map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;"); map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;"); map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;"); map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;"); map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;"); map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;"); map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;"); map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;"); map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;"); map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;"); map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;"); map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;"); map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;"); map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;"); map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;"); map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;"); map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;"); map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;"); map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;"); map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;"); map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;"); map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;"); map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;"); map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;"); map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;"); map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;"); map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;"); map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;"); map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;"); map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;"); map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;"); map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;"); map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;"); map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;"); map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;"); map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;"); map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;"); map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;"); map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;"); map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;"); map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;"); map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;"); map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map); HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = List.copyOf(map.keySet()); HTML_ENTITIES = List.copyOf(map.keySet());
} }
} }

75
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -2,55 +2,54 @@ package com.commafeed.backend.feed.parser;
import java.text.Bidi; import java.text.Bidi;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector { public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+"); private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*"); private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D; private static final double RTL_THRESHOLD = 0.4D;
public enum Direction { public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT LEFT_TO_RIGHT,
} RIGHT_TO_LEFT
}
public static Direction detect(String input) { public static Direction detect(String input) {
if (input == null || input.isBlank()) { if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT; return Direction.LEFT_TO_RIGHT;
} }
long rtl = 0; long rtl = 0;
long total = 0; long total = 0;
for (String token : WORDS_PATTERN.split(input)) { for (String token : WORDS_PATTERN.split(input)) {
// skip urls // skip urls
if (URL_PATTERN.matcher(token).matches()) { if (URL_PATTERN.matcher(token).matches()) {
continue; continue;
} }
// skip numbers // skip numbers
if (NumberUtils.isCreatable(token)) { if (NumberUtils.isCreatable(token)) {
continue; continue;
} }
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length()); boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) { if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT); Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) { if (bidi.getBaseLevel() == 1) {
rtl++; rtl++;
} }
} }
total++; total++;
} }
if (total == 0) { if (total == 0) {
return Direction.LEFT_TO_RIGHT; return Direction.LEFT_TO_RIGHT;
} }
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
} }

107
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java
View File

@@ -1,10 +1,8 @@
package com.commafeed.backend.feed.parser; package com.commafeed.backend.feed.parser;
import jakarta.inject.Singleton;
import java.util.Collection; import java.util.Collection;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit; import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie; import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@@ -13,70 +11,71 @@ import org.jdom2.Verifier;
@Singleton @Singleton
public class XMLCleaner { public class XMLCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE); private static final Pattern DOCTYPE_PATTERN =
Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private final Trie trie = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build(); private final Trie trie =
Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
public String clean(String xml) { public String clean(String xml) {
xml = removeCharactersBeforeFirstXmlTag(xml); xml = removeCharactersBeforeFirstXmlTag(xml);
xml = removeInvalidXmlCharacters(xml); xml = removeInvalidXmlCharacters(xml);
xml = replaceHtmlEntitiesWithNumericEntities(xml); xml = replaceHtmlEntitiesWithNumericEntities(xml);
xml = removeDoctypeDeclarations(xml); xml = removeDoctypeDeclarations(xml);
return xml; return xml;
} }
String removeCharactersBeforeFirstXmlTag(String xml) { String removeCharactersBeforeFirstXmlTag(String xml) {
if (StringUtils.isBlank(xml)) { if (StringUtils.isBlank(xml)) {
return null; return null;
} }
int pos = xml.indexOf('<'); int pos = xml.indexOf('<');
return pos < 0 ? null : xml.substring(pos); return pos < 0 ? null : xml.substring(pos);
} }
String removeInvalidXmlCharacters(String xml) { String removeInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) { if (StringUtils.isBlank(xml)) {
return null; return null;
} }
return xml.codePoints() return xml.codePoints()
.filter(Verifier::isXMLCharacter) .filter(Verifier::isXMLCharacter)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString(); .toString();
} }
// https://stackoverflow.com/a/40836618 // https://stackoverflow.com/a/40836618
String replaceHtmlEntitiesWithNumericEntities(String source) { String replaceHtmlEntitiesWithNumericEntities(String source) {
if (StringUtils.isBlank(source)) { if (StringUtils.isBlank(source)) {
return null; return null;
} }
// Create a buffer sufficiently large that re-allocations are minimized. // Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1); StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = trie.parseText(source); Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0; int prevIndex = 0;
for (Emit emit : emits) { for (Emit emit : emits) {
int matchIndex = emit.getStart(); int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex); sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword())); sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1; prevIndex = emit.getEnd() + 1;
} }
// Add the remainder of the string (contains no more matches). // Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex)); sb.append(source.substring(prevIndex));
return sb.toString(); return sb.toString();
} }
String removeDoctypeDeclarations(String xml) { String removeDoctypeDeclarations(String xml) {
if (StringUtils.isBlank(xml)) { if (StringUtils.isBlank(xml)) {
return null; return null;
} }
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
} }

27
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
View File

@@ -1,33 +1,28 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue; import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType; import jakarta.persistence.GenerationType;
import jakarta.persistence.Id; import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass; import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator; import jakarta.persistence.TableGenerator;
import java.io.Serializable;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
/** /** Abstract model for all entities, defining id and table generator */
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial") @SuppressWarnings("serial")
@MappedSuperclass @MappedSuperclass
@Getter @Getter
@Setter @Setter
public abstract class AbstractModel implements Serializable { public abstract class AbstractModel implements Serializable {
@Id @Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen") @GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator( @TableGenerator(
name = "gen", name = "gen",
table = "hibernate_sequences", table = "hibernate_sequences",
pkColumnName = "sequence_name", pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value", valueColumnName = "sequence_next_hi_value",
allocationSize = 1000) allocationSize = 1000)
private Long id; private Long id;
} }

134
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.Lob; import jakarta.persistence.Lob;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import java.sql.Types;
import org.hibernate.annotations.JdbcTypeCode; import java.time.Instant;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity @Entity
@Table(name = "FEEDS") @Table(name = "FEEDS")
@@ -20,97 +17,66 @@ import lombok.Setter;
@Setter @Setter
public class Feed extends AbstractModel { public class Feed extends AbstractModel {
/** /** The url of the feed */
* The url of the feed @Lob
*/ @Column(length = Integer.MAX_VALUE, nullable = false)
@Lob @JdbcTypeCode(Types.LONGVARCHAR)
@Column(length = Integer.MAX_VALUE, nullable = false) private String url;
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/** /** cache the url after potential http 30x redirects */
* cache the url after potential http 30x redirects @Column(name = "url_after_redirect", length = 2048, nullable = false)
*/ private String urlAfterRedirect;
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false) @Column(length = 2048, nullable = false)
private String normalizedUrl; private String normalizedUrl;
@Column(length = 40, nullable = false) @Column(length = 40, nullable = false)
private String normalizedUrlHash; private String normalizedUrlHash;
/** /** The url of the website, extracted from the feed */
* The url of the website, extracted from the feed @Lob
*/ @Column(length = Integer.MAX_VALUE)
@Lob @JdbcTypeCode(Types.LONGVARCHAR)
@Column(length = Integer.MAX_VALUE) private String link;
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
@Lob @Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE) @Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR) @JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl; private String iconUrl;
/** /** Last time we tried to fetch the feed */
* Last time we tried to fetch the feed @Column private Instant lastUpdated;
*/
@Column
private Instant lastUpdated;
/** /** Last publishedDate value in the feed */
* Last publishedDate value in the feed @Column private Instant lastPublishedDate;
*/
@Column
private Instant lastPublishedDate;
/** /** date of the last entry of the feed */
* date of the last entry of the feed @Column private Instant lastEntryDate;
*/
@Column
private Instant lastEntryDate;
/** /** error message while retrieving the feed */
* error message while retrieving the feed @Lob
*/ @Column(length = Integer.MAX_VALUE)
@Lob @JdbcTypeCode(Types.LONGVARCHAR)
@Column(length = Integer.MAX_VALUE) private String message;
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/** /** times we failed to retrieve the feed */
* times we failed to retrieve the feed private int errorCount;
*/
private int errorCount;
/** /** feed refresh is disabled until this date */
* feed refresh is disabled until this date @Column private Instant disabledUntil;
*/
@Column
private Instant disabledUntil;
/** /** http header returned by the feed */
* http header returned by the feed @Column(length = 64)
*/ private String lastModifiedHeader;
@Column(length = 64)
private String lastModifiedHeader;
/** /** http header returned by the feed */
* http header returned by the feed @Column(length = 255)
*/ private String etagHeader;
@Column(length = 255)
private String etagHeader;
/** /** average time between entries in the feed */
* average time between entries in the feed private Long averageEntryInterval;
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
/** last hash of the content of the feed xml */
@Column(length = 40)
private String lastContentHash;
} }

20
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
View File

@@ -6,7 +6,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn; import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -17,18 +16,17 @@ import lombok.Setter;
@Setter @Setter
public class FeedCategory extends AbstractModel { public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false) @Column(length = 128, nullable = false)
private String name; private String name;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false) @JoinColumn(nullable = false)
private User user; private User user;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent; private FeedCategory parent;
private boolean collapsed; private boolean collapsed;
private int position;
private int position;
} }

53
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
View File

@@ -1,8 +1,5 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType; import jakarta.persistence.CascadeType;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
@@ -11,7 +8,8 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany; import jakarta.persistence.OneToMany;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import java.time.Instant;
import java.util.Set;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -22,39 +20,32 @@ import lombok.Setter;
@Setter @Setter
public class FeedEntry extends AbstractModel { public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false) @Column(length = 2048, nullable = false)
private String guid; private String guid;
@Column(length = 40, nullable = false) @Column(length = 40, nullable = false)
private String guidHash; private String guidHash;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
private Feed feed; private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false) @ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false) @JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content; private FeedEntryContent content;
@Column(length = 2048) @Column(length = 2048)
private String url; private String url;
/** /** the moment the entry was inserted in the database */
* the moment the entry was inserted in the database @Column private Instant inserted;
*/
@Column
private Instant inserted;
/** /** the moment the entry was published in the feed */
* the moment the entry was published in the feed @Column(name = "updated")
* private Instant published;
*/
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE) @OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses; private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
} }

119
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
View File

@@ -1,21 +1,17 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.sql.Types; import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.EnumType; import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated; import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob; import jakarta.persistence.Lob;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import java.sql.Types;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
@Entity @Entity
@Table(name = "FEEDENTRYCONTENTS") @Table(name = "FEEDENTRYCONTENTS")
@@ -24,73 +20,74 @@ import lombok.Setter;
@Setter @Setter
public class FeedEntryContent extends AbstractModel { public class FeedEntryContent extends AbstractModel {
public enum Direction { public enum Direction {
@JsonProperty("ltr") @JsonProperty("ltr")
LTR, LTR,
@JsonProperty("rtl") @JsonProperty("rtl")
RTL, RTL,
@JsonProperty("unknown") @JsonProperty("unknown")
UNKNOWN UNKNOWN
} }
@Column(length = 2048) @Column(length = 2048)
private String title; private String title;
@Column(length = 40) @Column(length = 40)
private String titleHash; private String titleHash;
@Lob @Lob
@Column(length = Integer.MAX_VALUE) @Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR) @JdbcTypeCode(Types.LONGVARCHAR)
private String content; private String content;
@Column(length = 40) @Column(length = 40)
private String contentHash; private String contentHash;
@Column(name = "author", length = 128) @Column(name = "author", length = 128)
private String author; private String author;
@Column(length = 2048) @Column(length = 2048)
private String enclosureUrl; private String enclosureUrl;
@Column(length = 255) @Column(length = 255)
private String enclosureType; private String enclosureType;
@Lob @Lob
@Column(length = Integer.MAX_VALUE) @Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR) @JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription; private String mediaDescription;
@Column(length = 2048) @Column(length = 2048)
private String mediaThumbnailUrl; private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth; private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight; private Integer mediaThumbnailHeight;
@Column(length = 4096) @Column(length = 4096)
private String categories; private String categories;
@Column @Column
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
private Direction direction = Direction.UNKNOWN; private Direction direction = Direction.UNKNOWN;
public boolean equivalentTo(FeedEntryContent c) { public boolean equivalentTo(FeedEntryContent c) {
if (c == null) { if (c == null) {
return false; return false;
} }
return new EqualsBuilder().append(title, c.title) return new EqualsBuilder()
.append(content, c.content) .append(title, c.title)
.append(author, c.author) .append(content, c.content)
.append(categories, c.categories) .append(author, c.author)
.append(enclosureUrl, c.enclosureUrl) .append(categories, c.categories)
.append(enclosureType, c.enclosureType) .append(enclosureUrl, c.enclosureUrl)
.append(mediaDescription, c.mediaDescription) .append(enclosureType, c.enclosureType)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl) .append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth) .append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight) .append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.build(); .append(mediaThumbnailHeight, c.mediaThumbnailHeight)
} .build();
}
} }

68
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
View File

@@ -1,9 +1,5 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.FetchType; import jakarta.persistence.FetchType;
@@ -11,7 +7,9 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import jakarta.persistence.Transient; import jakarta.persistence.Transient;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -22,48 +20,40 @@ import lombok.Setter;
@Setter @Setter
public class FeedEntryStatus extends AbstractModel { public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false) @JoinColumn(nullable = false)
private FeedSubscription subscription; private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false) @JoinColumn(nullable = false)
private FeedEntry entry; private FeedEntry entry;
@Column(name = "read_status") @Column(name = "read_status")
private boolean read; private boolean read;
private boolean starred;
@Transient private boolean starred;
private boolean markable;
@Transient @Transient private boolean markable;
private List<FeedEntryTag> tags = new ArrayList<>();
/** @Transient private List<FeedEntryTag> tags = new ArrayList<>();
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY) /** Denormalization starts here */
@JoinColumn(nullable = false) @ManyToOne(fetch = FetchType.LAZY)
private User user; @JoinColumn(nullable = false)
private User user;
@Column @Column private Instant entryInserted;
private Instant entryInserted;
@Column(name = "entryUpdated") @Column(name = "entryUpdated")
private Instant entryPublished; private Instant entryPublished;
public FeedEntryStatus() { public FeedEntryStatus() {}
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
} }

31
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java
View File

@@ -6,7 +6,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn; import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -17,24 +16,22 @@ import lombok.Setter;
@Setter @Setter
public class FeedEntryTag extends AbstractModel { public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id") @JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
private User user; private User user;
@JoinColumn(name = "entry_id") @JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry; private FeedEntry entry;
@Column(name = "name", length = 40) @Column(name = "name", length = 40)
private String name; private String name;
public FeedEntryTag() { public FeedEntryTag() {}
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
} }

47
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java
View File

@@ -1,7 +1,5 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType; import jakarta.persistence.CascadeType;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
@@ -10,7 +8,7 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne; import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany; import jakarta.persistence.OneToMany;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import java.util.Set;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -21,35 +19,34 @@ import lombok.Setter;
@Setter @Setter
public class FeedSubscription extends AbstractModel { public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false) @JoinColumn(nullable = false)
private User user; private User user;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false) @JoinColumn(nullable = false)
private Feed feed; private Feed feed;
@Column(length = 128, nullable = false) @Column(length = 128, nullable = false)
private String title; private String title;
@ManyToOne(fetch = FetchType.LAZY) @ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category; private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE) @OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses; private Set<FeedEntryStatus> statuses;
private int position; private int position;
@Column(name = "filtering_expression", length = 4096) @Column(name = "filtering_expression", length = 4096)
private String filter; private String filter;
@Column(name = "filtering_expression_legacy", length = 4096) @Column(name = "filtering_expression_legacy", length = 4096)
private String filterLegacy; private String filterLegacy;
@Column(name = "push_notifications_enabled") @Column(name = "push_notifications_enabled")
private boolean pushNotificationsEnabled; private boolean pushNotificationsEnabled;
@Column(name = "auto_mark_as_read_after_days")
private Integer autoMarkAsReadAfterDays;
@Column(name = "auto_mark_as_read_after_days")
private Integer autoMarkAsReadAfterDays;
} }

47
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java
View File

@@ -2,40 +2,35 @@ package com.commafeed.backend.model;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import lombok.experimental.UtilityClass;
import org.hibernate.Hibernate; import org.hibernate.Hibernate;
import org.hibernate.HibernateException; import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy; import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer; import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
public class Models { public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH public static final Instant MINIMUM_INSTANT =
// mariadb timestamp range starts at 1970-01-01 00:00:01 Instant.EPOCH
.plusSeconds(1) // mariadb timestamp range starts at 1970-01-01 00:00:01
// make sure the timestamp fits for all timezones .plusSeconds(1)
.plus(Duration.ofHours(24)); // make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/** /** initialize a proxy */
* initialize a proxy public static void initialize(Object proxy) throws HibernateException {
*/ Hibernate.initialize(proxy);
public static void initialize(Object proxy) throws HibernateException { }
Hibernate.initialize(proxy);
}
/** /** extract the id from the proxy without initializing it */
* extract the id from the proxy without initializing it public static Long getId(AbstractModel model) {
*/ if (model instanceof HibernateProxy proxy) {
public static Long getId(AbstractModel model) { LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (model instanceof HibernateProxy proxy) { if (lazyInitializer.isUninitialized()) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer(); return (Long) lazyInitializer.getIdentifier();
if (lazyInitializer.isUninitialized()) { }
return (Long) lazyInitializer.getIdentifier(); }
} return model.getId();
} }
return model.getId();
}
} }

57
commafeed-server/src/main/java/com/commafeed/backend/model/User.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.Lob; import jakarta.persistence.Lob;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import java.sql.Types;
import org.hibernate.annotations.JdbcTypeCode; import java.time.Instant;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity @Entity
@Table(name = "USERS") @Table(name = "USERS")
@@ -20,40 +17,36 @@ import lombok.Setter;
@Setter @Setter
public class User extends AbstractModel { public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true) @Column(length = 32, nullable = false, unique = true)
private String name; private String name;
@Column(length = 255, unique = true) @Column(length = 255, unique = true)
private String email; private String email;
@Lob @Lob
@Column(length = Integer.MAX_VALUE, nullable = false) @Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY) @JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password; private byte[] password;
@Column(length = 40, unique = true) @Column(length = 40, unique = true)
private String apiKey; private String apiKey;
@Lob @Lob
@Column(length = Integer.MAX_VALUE, nullable = false) @Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY) @JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt; private byte[] salt;
@Column(nullable = false) @Column(nullable = false)
private boolean disabled; private boolean disabled;
@Column @Column private Instant lastLogin;
private Instant lastLogin;
@Column @Column private Instant created;
private Instant created;
@Column(length = 40) @Column(length = 40)
private String recoverPasswordToken; private String recoverPasswordToken;
@Column @Column private Instant recoverPasswordTokenDate;
private Instant recoverPasswordTokenDate;
@Column @Column private Instant lastForceRefresh;
private Instant lastForceRefresh;
} }

33
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java
View File

@@ -8,7 +8,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn; import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne; import jakarta.persistence.OneToOne;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@@ -19,25 +18,23 @@ import lombok.Setter;
@Setter @Setter
public class UserRole extends AbstractModel { public class UserRole extends AbstractModel {
public enum Role { public enum Role {
USER, ADMIN USER,
} ADMIN
}
@OneToOne(fetch = FetchType.LAZY) @OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false) @JoinColumn(name = "user_id", nullable = false)
private User user; private User user;
@Column(name = "roleName", nullable = false) @Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
private Role role; private Role role;
public UserRole() { public UserRole() {}
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
} }

241
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java
View File

@@ -1,8 +1,6 @@
package com.commafeed.backend.model; package com.commafeed.backend.model;
import java.io.Serializable; import com.fasterxml.jackson.annotation.JsonProperty;
import java.sql.Types;
import jakarta.persistence.Column; import jakarta.persistence.Column;
import jakarta.persistence.Embeddable; import jakarta.persistence.Embeddable;
import jakarta.persistence.Embedded; import jakarta.persistence.Embedded;
@@ -14,13 +12,11 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob; import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne; import jakarta.persistence.OneToOne;
import jakarta.persistence.Table; import jakarta.persistence.Table;
import java.io.Serializable;
import org.hibernate.annotations.JdbcTypeCode; import java.sql.Types;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity @Entity
@Table(name = "USERSETTINGS") @Table(name = "USERSETTINGS")
@@ -29,155 +25,154 @@ import lombok.Setter;
@Setter @Setter
public class UserSettings extends AbstractModel { public class UserSettings extends AbstractModel {
public enum ReadingMode { public enum ReadingMode {
@JsonProperty("all") @JsonProperty("all")
ALL, ALL,
@JsonProperty("unread") @JsonProperty("unread")
UNREAD; UNREAD;
// method called for query parameters // method called for query parameters
public static ReadingMode fromString(final String s) { public static ReadingMode fromString(final String s) {
return ReadingMode.valueOf(s.toUpperCase()); return ReadingMode.valueOf(s.toUpperCase());
} }
} }
public enum ReadingOrder { public enum ReadingOrder {
@JsonProperty("asc") @JsonProperty("asc")
ASC, ASC,
@JsonProperty("desc") @JsonProperty("desc")
DESC; DESC;
// method called for query parameters // method called for query parameters
public static ReadingOrder fromString(final String s) { public static ReadingOrder fromString(final String s) {
return ReadingOrder.valueOf(s.toUpperCase()); return ReadingOrder.valueOf(s.toUpperCase());
} }
} }
public enum ScrollMode { public enum ScrollMode {
@JsonProperty("always") @JsonProperty("always")
ALWAYS, ALWAYS,
@JsonProperty("never") @JsonProperty("never")
NEVER, NEVER,
@JsonProperty("if_needed") @JsonProperty("if_needed")
IF_NEEDED IF_NEEDED
} }
public enum IconDisplayMode { public enum IconDisplayMode {
@JsonProperty("always") @JsonProperty("always")
ALWAYS, ALWAYS,
@JsonProperty("never") @JsonProperty("never")
NEVER, NEVER,
@JsonProperty("on_desktop") @JsonProperty("on_desktop")
ON_DESKTOP, ON_DESKTOP,
@JsonProperty("on_mobile") @JsonProperty("on_mobile")
ON_MOBILE ON_MOBILE
} }
public enum PushNotificationType { public enum PushNotificationType {
@JsonProperty("ntfy") @JsonProperty("ntfy")
NTFY, NTFY,
@JsonProperty("gotify") @JsonProperty("gotify")
GOTIFY, GOTIFY,
@JsonProperty("pushover") @JsonProperty("pushover")
PUSHOVER PUSHOVER
} }
@OneToOne(fetch = FetchType.LAZY) @OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true) @JoinColumn(name = "user_id", nullable = false, unique = true)
private User user; private User user;
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
@Column(nullable = false) @Column(nullable = false)
private ReadingMode readingMode; private ReadingMode readingMode;
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
@Column(nullable = false) @Column(nullable = false)
private ReadingOrder readingOrder; private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4) @Column(name = "user_lang", length = 4)
private String language; private String language;
private boolean showRead; private boolean showRead;
private boolean scrollMarks; private boolean scrollMarks;
@Lob @Lob
@Column(length = Integer.MAX_VALUE) @Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR) @JdbcTypeCode(Types.LONGVARCHAR)
private String customCss; private String customCss;
@Lob @Lob
@Column(length = Integer.MAX_VALUE) @Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR) @JdbcTypeCode(Types.LONGVARCHAR)
private String customJs; private String customJs;
@Column(name = "scroll_speed") @Column(name = "scroll_speed")
private int scrollSpeed; private int scrollSpeed;
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
@Column(nullable = false) @Column(nullable = false)
private ScrollMode scrollMode; private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling; private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
@Column(nullable = false) @Column(nullable = false)
private IconDisplayMode starIconDisplayMode; private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
@Column(nullable = false) @Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode; private IconDisplayMode externalLinkIconDisplayMode;
@Column(name = "primary_color", length = 32) @Column(name = "primary_color", length = 32)
private String primaryColor; private String primaryColor;
private boolean markAllAsReadConfirmation; private boolean markAllAsReadConfirmation;
private boolean markAllAsReadNavigateToNextUnread; private boolean markAllAsReadNavigateToNextUnread;
private boolean customContextMenu; private boolean customContextMenu;
private boolean mobileFooter; private boolean mobileFooter;
private boolean unreadCountTitle; private boolean unreadCountTitle;
private boolean unreadCountFavicon; private boolean unreadCountFavicon;
private boolean disablePullToRefresh; private boolean disablePullToRefresh;
private boolean email; private boolean email;
private boolean gmail; private boolean gmail;
private boolean facebook; private boolean facebook;
private boolean twitter; private boolean twitter;
private boolean tumblr; private boolean tumblr;
private boolean instapaper; private boolean instapaper;
private boolean buffer; private boolean buffer;
@Embedded @Embedded
private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings(); private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embeddable @Embeddable
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Getter @Getter
@Setter @Setter
public static class PushNotificationUserSettings implements Serializable { public static class PushNotificationUserSettings implements Serializable {
@Enumerated(EnumType.STRING) @Enumerated(EnumType.STRING)
@Column(name = "push_notification_type", length = 16) @Column(name = "push_notification_type", length = 16)
private PushNotificationType type; private PushNotificationType type;
@Column(name = "push_notification_server_url", length = 1024) @Column(name = "push_notification_server_url", length = 1024)
private String serverUrl; private String serverUrl;
@Column(name = "push_notification_user_id", length = 512) @Column(name = "push_notification_user_id", length = 512)
private String userId; private String userId;
@Column(name = "push_notification_user_secret", length = 512) @Column(name = "push_notification_user_secret", length = 512)
private String userSecret; private String userSecret;
@Column(name = "push_notification_topic", length = 256)
private String topic;
}
@Column(name = "push_notification_topic", length = 256)
private String topic;
}
} }

129
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
View File

@@ -1,13 +1,5 @@
package com.commafeed.backend.opml; package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO; import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory; import com.commafeed.backend.model.FeedCategory;
@@ -16,75 +8,88 @@ import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute; import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml; import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline; import com.rometools.opml.feed.opml.Outline;
import jakarta.inject.Singleton;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.ObjectUtils;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class OPMLExporter { public class OPMLExporter {
private static final Comparator<FeedCategory> CATEGORY_COMPARATOR = Comparator private static final Comparator<FeedCategory> CATEGORY_COMPARATOR =
.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)); Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedSubscription> SUBSCRIPTION_COMPARATOR = Comparator private static final Comparator<FeedSubscription> SUBSCRIPTION_COMPARATOR =
.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)); Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private final FeedCategoryDAO feedCategoryDAO; private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO; private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) { public Opml export(User user) {
Opml opml = new Opml(); Opml opml = new Opml();
opml.setFeedType("opml_1.0"); opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName())); opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date()); opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user); List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(CATEGORY_COMPARATOR); categories.sort(CATEGORY_COMPARATOR);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user); List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(SUBSCRIPTION_COMPARATOR); subscriptions.sort(SUBSCRIPTION_COMPARATOR);
// export root categories // export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) { for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions)); opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
} }
// export root subscriptions // export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) { for (FeedSubscription sub :
opml.getOutlines().add(buildSubscriptionOutline(sub)); subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
} opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml; return opml;
}
} private Outline buildCategoryOutline(
FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) { for (FeedCategory child :
Outline outline = new Outline(); categories.stream()
outline.setText(cat.getName()); .filter(
outline.setTitle(cat.getName()); c ->
c.getParent() != null
&& c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedCategory child : categories.stream() for (FeedSubscription sub :
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId())) subscriptions.stream()
.toList()) { .filter(
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions)); s ->
} s.getCategory() != null
&& s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
for (FeedSubscription sub : subscriptions.stream() private Outline buildSubscriptionOutline(FeedSubscription sub) {
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId())) Outline outline = new Outline();
.toList()) { outline.setText(sub.getTitle());
outline.getChildren().add(buildSubscriptionOutline(sub)); outline.setTitle(sub.getTitle());
} outline.setType("rss");
return outline; outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
} if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
private Outline buildSubscriptionOutline(FeedSubscription sub) { }
Outline outline = new Outline(); return outline;
outline.setText(sub.getTitle()); }
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
} }

121
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
View File

@@ -1,13 +1,5 @@
package com.commafeed.backend.opml; package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO; import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.XMLCleaner; import com.commafeed.backend.feed.parser.XMLCleaner;
@@ -18,71 +10,76 @@ import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline; import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException; import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput; import com.rometools.rome.io.WireFeedInput;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.util.List;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class OPMLImporter { public class OPMLImporter {
private final XMLCleaner xmlCleaner; private final XMLCleaner xmlCleaner;
private final FeedCategoryDAO feedCategoryDAO; private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService; private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException { public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xmlCleaner.clean(xml); xml = xmlCleaner.clean(xml);
if (xml == null) { if (xml == null) {
throw new IllegalArgumentException("Invalid OPML"); throw new IllegalArgumentException("Invalid OPML");
} }
WireFeedInput input = new WireFeedInput(); WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml)); Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines(); List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) { for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i); handleOutline(user, outlines.get(i), null, i);
} }
} }
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) { private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren(); List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) { if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128); String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) { if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128); name = FeedUtils.truncate(outline.getTitle(), 128);
} }
FeedCategory category = feedCategoryDAO.findByName(user, name, parent); FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) { if (category == null) {
if (StringUtils.isBlank(name)) { if (StringUtils.isBlank(name)) {
name = "Unnamed category"; name = "Unnamed category";
} }
category = new FeedCategory(); category = new FeedCategory();
category.setName(name); category.setName(name);
category.setParent(parent); category.setParent(parent);
category.setUser(user); category.setUser(user);
category.setPosition(position); category.setPosition(position);
feedCategoryDAO.persist(category); feedCategoryDAO.persist(category);
} }
for (int i = 0; i < children.size(); i++) { for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i); handleOutline(user, children.get(i), category, i);
} }
} else { } else {
String name = FeedUtils.truncate(outline.getText(), 128); String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) { if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128); name = FeedUtils.truncate(outline.getTitle(), 128);
} }
if (StringUtils.isBlank(name)) { if (StringUtils.isBlank(name)) {
name = "Unnamed subscription"; name = "Unnamed subscription";
} }
// make sure we continue with the import process even if a feed failed // make sure we continue with the import process even if a feed failed
try { try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position); feedSubscriptionService.subscribe(
} catch (Exception e) { user, outline.getXmlUrl(), name, parent, position);
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage()); } catch (Exception e) {
} log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
} }
} }
}
} }

42
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
View File

@@ -1,38 +1,32 @@
package com.commafeed.backend.rome; package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser; import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed; import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException; import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
/** /** Support for OPML 1.1 parsing */
* Support for OPML 1.1 parsing
*
*/
@RegisterForReflection @RegisterForReflection
public class OPML11Parser extends OPML10Parser { public class OPML11Parser extends OPML10Parser {
public OPML11Parser() { public OPML11Parser() {
super("opml_1.1"); super("opml_1.1");
} }
@Override @Override
public boolean isMyType(Document document) { public boolean isMyType(Document document) {
Element e = document.getRootElement(); Element e = document.getRootElement();
return e.getName().equals("opml"); return e.getName().equals("opml");
}
} @Override
public WireFeed parse(Document document, boolean validate, Locale locale)
@Override throws IllegalArgumentException, FeedException {
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException { document.getRootElement().getChildren().add(new Element("head"));
document.getRootElement().getChildren().add(new Element("head")); return super.parse(document, validate, locale);
return super.parse(document, validate, locale); }
}
} }

317
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -1,12 +1,14 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import jakarta.inject.Singleton;
import java.io.StringReader; import java.io.StringReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import lombok.RequiredArgsConstructor;
import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
@@ -21,155 +23,212 @@ import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource; import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration; import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedEntryContentCleaningService { public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList(); private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border"); private static final List<String> ALLOWED_IFRAME_CSS_RULES =
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height"); Arrays.asList("height", "width", "border");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' }; private static final List<String> ALLOWED_IMG_CSS_RULES =
Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] {'(', ')'};
public String clean(String content, String baseUri, boolean keepTextOnly) { public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) { if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri); baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri); Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST); Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty); Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) { for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style"); String style = e.attr("style");
String escaped = escapeIFrameCss(style); String escaped = escapeIFrameCss(style);
e.attr("style", escaped); e.attr("style", escaped);
} }
for (Element e : clean.select("img[style]")) { for (Element e : clean.select("img[style]")) {
String style = e.attr("style"); String style = e.attr("style");
String escaped = escapeImgCss(style); String escaped = escapeImgCss(style);
e.attr("style", escaped); e.attr("style", escaped);
} }
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false)); clean.outputSettings(
Element body = clean.body(); new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
if (keepTextOnly) { Element body = clean.body();
content = body.text(); if (keepTextOnly) {
} else { content = body.text();
content = body.html(); } else {
} content = body.html();
} }
return content; }
} return content;
}
private static Safelist buildWhiteList() { private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist(); Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1", whitelist.addTags(
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup", "a",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul"); "b",
"blockquote",
"br",
"caption",
"cite",
"code",
"col",
"colgroup",
"dd",
"div",
"dl",
"dt",
"em",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"i",
"iframe",
"img",
"li",
"ol",
"p",
"pre",
"q",
"small",
"strike",
"strong",
"sub",
"sup",
"table",
"tbody",
"td",
"tfoot",
"th",
"thead",
"tr",
"u",
"ul");
whitelist.addAttributes("div", "dir"); whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir"); whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir"); whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir"); whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir"); whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title"); whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite"); whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width"); whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width"); whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style"); whitelist.addAttributes(
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style"); "iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("ol", "start", "type"); whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("q", "cite"); whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width"); whitelist.addAttributes("q", "cite");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width"); whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width"); whitelist.addAttributes(
whitelist.addAttributes("ul", "type"); "td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes(
"th",
"border",
"bordercolor",
"abbr",
"axis",
"colspan",
"rowspan",
"scope",
"width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto"); whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https"); whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https"); whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https"); whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank"); whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer"); whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist; return whitelist;
} }
private String escapeIFrameCss(String orig) { private String escapeIFrameCss(String orig) {
String rule = ""; String rule = "";
try { try {
List<String> rules = new ArrayList<>(); List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig))); CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) { for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i); String property = decl.item(i);
String value = decl.getPropertyValue(property); String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) { if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue; continue;
} }
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) { if (ALLOWED_IFRAME_CSS_RULES.contains(property)
rules.add(property + ":" + decl.getPropertyValue(property) + ";"); && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
} rules.add(property + ":" + decl.getPropertyValue(property) + ";");
} }
rule = StringUtils.join(rules, ""); }
} catch (Exception e) { rule = StringUtils.join(rules, "");
log.error(e.getMessage(), e); } catch (Exception e) {
} log.error(e.getMessage(), e);
return rule; }
} return rule;
}
private String escapeImgCss(String orig) { private String escapeImgCss(String orig) {
String rule = ""; String rule = "";
try { try {
List<String> rules = new ArrayList<>(); List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig))); CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) { for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i); String property = decl.item(i);
String value = decl.getPropertyValue(property); String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) { if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue; continue;
} }
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) { if (ALLOWED_IMG_CSS_RULES.contains(property)
rules.add(property + ":" + decl.getPropertyValue(property) + ";"); && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
} rules.add(property + ":" + decl.getPropertyValue(property) + ";");
} }
rule = StringUtils.join(rules, ""); }
} catch (Exception e) { rule = StringUtils.join(rules, "");
log.error(e.getMessage(), e); } catch (Exception e) {
} log.error(e.getMessage(), e);
return rule; }
} return rule;
}
private CSSOMParser buildCssParser() { private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21()); CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() { parser.setErrorHandler(
@Override new ErrorHandler() {
public void warning(CSSParseException exception) throws CSSException { @Override
log.debug("warning while parsing css: {}", exception.getMessage(), exception); public void warning(CSSParseException exception) throws CSSException {
} log.debug(
"warning while parsing css: {}", exception.getMessage(), exception);
}
@Override @Override
public void error(CSSParseException exception) throws CSSException { public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception); log.debug("error while parsing css: {}", exception.getMessage(), exception);
} }
@Override @Override
public void fatalError(CSSParseException exception) throws CSSException { public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception); log.debug(
} "fatal error while parsing css: {}",
}); exception.getMessage(),
exception);
}
});
return parser; return parser;
} }
} }

100
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -1,11 +1,5 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests; import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO; import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
@@ -13,59 +7,65 @@ import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure; import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media; import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent; import com.commafeed.backend.model.FeedEntryContent;
import jakarta.inject.Singleton;
import java.util.Optional;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class FeedEntryContentService { public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService; private final FeedEntryContentCleaningService cleaningService;
/** /** this is NOT thread-safe */
* this is NOT thread-safe public FeedEntryContent findOrCreate(Content content, String baseUrl) {
*/ FeedEntryContent entryContent = buildContent(content, baseUrl);
public FeedEntryContent findOrCreate(Content content, String baseUrl) { Optional<FeedEntryContent> existing =
FeedEntryContent entryContent = buildContent(content, baseUrl); feedEntryContentDAO
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash()) .findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream() .stream()
.filter(entryContent::equivalentTo) .filter(entryContent::equivalentTo)
.findFirst(); .findFirst();
if (existing.isPresent()) { if (existing.isPresent()) {
return existing.get(); return existing.get();
} else { } else {
feedEntryContentDAO.persist(entryContent); feedEntryContentDAO.persist(entryContent);
return entryContent; return entryContent;
} }
} }
private FeedEntryContent buildContent(Content content, String baseUrl) { private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent(); FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title()))); entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content()))); entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048)); entryContent.setTitle(
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false)); FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128)); entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096)); entryContent.setAuthor(
entryContent.setDirection( FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.RTL : FeedEntryContent.Direction.LTR); entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content())
? FeedEntryContent.Direction.RTL
: FeedEntryContent.Direction.LTR);
Enclosure enclosure = content.enclosure(); Enclosure enclosure = content.enclosure();
if (enclosure != null) { if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048)); entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type()); entryContent.setEnclosureType(enclosure.type());
} }
Media media = content.media(); Media media = content.media();
if (media != null) { if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false)); entryContent.setMediaDescription(
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048)); cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth()); entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailHeight(media.thumbnailHeight()); entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
} entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
return entryContent;
}
} }

166
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -1,22 +1,7 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelValidationException; import dev.cel.common.CelValidationException;
import dev.cel.common.types.SimpleType; import dev.cel.common.types.SimpleType;
@@ -25,83 +10,112 @@ import dev.cel.compiler.CelCompilerFactory;
import dev.cel.runtime.CelEvaluationException; import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime; import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory; import dev.cel.runtime.CelRuntimeFactory;
import jakarta.inject.Singleton;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class FeedEntryFilteringService { public class FeedEntryFilteringService {
private static final CelCompiler CEL_COMPILER = CelCompilerFactory.standardCelCompilerBuilder() private static final CelCompiler CEL_COMPILER =
.addVar("title", SimpleType.STRING) CelCompilerFactory.standardCelCompilerBuilder()
.addVar("titleLower", SimpleType.STRING) .addVar("title", SimpleType.STRING)
.addVar("author", SimpleType.STRING) .addVar("titleLower", SimpleType.STRING)
.addVar("authorLower", SimpleType.STRING) .addVar("author", SimpleType.STRING)
.addVar("content", SimpleType.STRING) .addVar("authorLower", SimpleType.STRING)
.addVar("contentLower", SimpleType.STRING) .addVar("content", SimpleType.STRING)
.addVar("url", SimpleType.STRING) .addVar("contentLower", SimpleType.STRING)
.addVar("urlLower", SimpleType.STRING) .addVar("url", SimpleType.STRING)
.addVar("categories", SimpleType.STRING) .addVar("urlLower", SimpleType.STRING)
.addVar("categoriesLower", SimpleType.STRING) .addVar("categories", SimpleType.STRING)
.build(); .addVar("categoriesLower", SimpleType.STRING)
private static final CelRuntime CEL_RUNTIME = CelRuntimeFactory.standardCelRuntimeBuilder().build(); .build();
private static final CelRuntime CEL_RUNTIME =
CelRuntimeFactory.standardCelRuntimeBuilder().build();
private final ExecutorService executor = Executors.newCachedThreadPool(); private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException { public boolean filterMatchesEntry(String filter, FeedEntry entry)
if (StringUtils.isBlank(filter)) { throws FeedEntryFilterException {
return true; if (StringUtils.isBlank(filter)) {
} return true;
}
String title = entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text(); String title =
String author = entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor(); entry.getContent().getTitle() == null
String content = entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text(); ? ""
String url = entry.getUrl() == null ? "" : entry.getUrl(); : Jsoup.parse(entry.getContent().getTitle()).text();
String categories = entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories(); String author =
entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String content =
entry.getContent().getContent() == null
? ""
: Jsoup.parse(entry.getContent().getContent()).text();
String url = entry.getUrl() == null ? "" : entry.getUrl();
String categories =
entry.getContent().getCategories() == null
? ""
: entry.getContent().getCategories();
Map<String, Object> data = new HashMap<>(); Map<String, Object> data = new HashMap<>();
data.put("title", title); data.put("title", title);
data.put("titleLower", title.toLowerCase()); data.put("titleLower", title.toLowerCase());
data.put("author", author); data.put("author", author);
data.put("authorLower", author.toLowerCase()); data.put("authorLower", author.toLowerCase());
data.put("content", content); data.put("content", content);
data.put("contentLower", content.toLowerCase()); data.put("contentLower", content.toLowerCase());
data.put("url", url); data.put("url", url);
data.put("urlLower", url.toLowerCase()); data.put("urlLower", url.toLowerCase());
data.put("categories", categories); data.put("categories", categories);
data.put("categoriesLower", categories.toLowerCase()); data.put("categoriesLower", categories.toLowerCase());
Future<Object> future = executor.submit(() -> evaluateCelExpression(filter, data)); Future<Object> future = executor.submit(() -> evaluateCelExpression(filter, data));
Object result; Object result;
try { try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS); result =
} catch (InterruptedException e) { future.get(
Thread.currentThread().interrupt(); config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(),
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e); TimeUnit.MILLISECONDS);
} catch (ExecutionException e) { } catch (InterruptedException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e); Thread.currentThread().interrupt();
} catch (TimeoutException e) { throw new FeedEntryFilterException(
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e); "interrupted while evaluating expression " + filter, e);
} } catch (ExecutionException e) {
throw new FeedEntryFilterException(
"Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
return Boolean.TRUE.equals(result); return Boolean.TRUE.equals(result);
} }
private Object evaluateCelExpression(String expression, Map<String, Object> data) private Object evaluateCelExpression(String expression, Map<String, Object> data)
throws CelValidationException, CelEvaluationException { throws CelValidationException, CelEvaluationException {
CelAbstractSyntaxTree ast = CEL_COMPILER.compile(expression).getAst(); CelAbstractSyntaxTree ast = CEL_COMPILER.compile(expression).getAst();
CelRuntime.Program program = CEL_RUNTIME.createProgram(ast); CelRuntime.Program program = CEL_RUNTIME.createProgram(ast);
return program.eval(data); return program.eval(data);
} }
@SuppressWarnings("serial") @SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception { public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) { public FeedEntryFilterException(String message, Throwable t) {
super(message, t); super(message, t);
} }
} }
} }

228
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -1,12 +1,5 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests; import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.FeedEntryStatusDAO;
@@ -20,7 +13,11 @@ import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException; import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -29,113 +26,146 @@ import lombok.extern.slf4j.Slf4j;
@Singleton @Singleton
public class FeedEntryService { public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO; private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO; private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO; private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService; private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService; private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) { public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid()); String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed); return feedEntryDAO.findExisting(guidHash, feed);
} }
public FeedEntry create(Feed feed, Entry entry) { public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry(); FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048)); feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid())); feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048)); feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published()); feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now()); feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed); feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink())); feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.persist(feedEntry); feedEntryDAO.persist(feedEntry);
return feedEntry; return feedEntry;
} }
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) { public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet()); Set<String> guidHashes =
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed); entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList(); Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
} return entries.stream()
.filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid())))
.toList();
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) { public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true; boolean matches = true;
try { try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry); matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) { } catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e); log.error("could not evaluate filter {}", sub.getFilter(), e);
} }
if (!matches) { if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry); FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true); status.setRead(true);
feedEntryStatusDAO.persist(status); feedEntryStatusDAO.persist(status);
} }
return matches; return matches;
} }
public void markEntry(User user, Long entryId, boolean read) { public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId); FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) { if (entry == null) {
return; return;
} }
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed()); FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) { if (sub == null) {
return; return;
} }
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry); FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) { if (status.isMarkable()) {
status.setRead(read); status.setRead(read);
feedEntryStatusDAO.merge(status); feedEntryStatusDAO.merge(status);
} }
} }
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) { public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId); FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) { if (sub == null) {
return; return;
} }
FeedEntry entry = feedEntryDAO.findById(entryId); FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) { if (entry == null) {
return; return;
} }
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry); FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred); status.setStarred(starred);
feedEntryStatusDAO.merge(status); feedEntryStatusDAO.merge(status);
} }
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore, public void markSubscriptionEntries(
List<FeedEntryKeyword> keywords) { User user,
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null, List<FeedSubscription> subscriptions,
false, null, null, null); Instant olderThan,
markList(statuses, olderThan, insertedBefore); Instant insertedBefore,
} List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findBySubscriptions(
user,
subscriptions,
true,
keywords,
null,
-1,
-1,
null,
false,
null,
null,
null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) { public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false); List<FeedEntryStatus> statuses =
markList(statuses, olderThan, insertedBefore); feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
} markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) { private void markList(
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> { List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
Instant entryDate = s.getEntry().getPublished(); List<FeedEntryStatus> statusesToMark =
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan); statuses.stream()
}).filter(s -> { .filter(FeedEntryStatus::isMarkable)
Instant insertedDate = s.getEntry().getInserted(); .filter(
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore); s -> {
}).toList(); Instant entryDate = s.getEntry().getPublished();
return olderThan == null
|| entryDate == null
|| entryDate.isBefore(olderThan);
})
.filter(
s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null
|| insertedDate == null
|| insertedDate.isBefore(insertedBefore);
})
.toList();
statusesToMark.forEach(s -> { statusesToMark.forEach(
s.setRead(true); s -> {
feedEntryStatusDAO.merge(s); s.setRead(true);
}); feedEntryStatusDAO.merge(s);
} });
}
} }

49
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -1,43 +1,42 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO; import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag; import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class FeedEntryTagService { public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO; private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO; private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) { public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId); FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) { if (entry == null) {
return; return;
} }
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry); List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet()); Set<String> existingTagNames =
existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream() List<FeedEntryTag> addList =
.filter(name -> !existingTagNames.contains(name)) tagNames.stream()
.map(name -> new FeedEntryTag(user, entry, name)) .filter(name -> !existingTagNames.contains(name))
.toList(); .map(name -> new FeedEntryTag(user, entry, name))
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList(); .toList();
List<FeedEntryTag> removeList =
addList.forEach(feedEntryTagDAO::persist); existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.delete(removeList);
}
addList.forEach(feedEntryTagDAO::persist);
feedEntryTagDAO.delete(removeList);
}
} }

99
commafeed-server/src/main/java/com/commafeed/backend/service/FeedFaviconService.java
View File

@@ -1,71 +1,72 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import org.apache.commons.lang3.ArrayUtils;
import com.commafeed.backend.favicon.Favicon; import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.favicon.FaviconFetcher; import com.commafeed.backend.favicon.FaviconFetcher;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.google.common.io.Resources; import com.google.common.io.Resources;
import io.quarkus.arc.All; import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
@Singleton @Singleton
@Slf4j @Slf4j
public class FeedFaviconService { public class FeedFaviconService {
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST = Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE); private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST =
private static final long MIN_ICON_LENGTH = 100; Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MAX_ICON_LENGTH = 100000; private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private final List<FaviconFetcher> faviconFetchers; private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon; private final Favicon defaultFavicon;
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException { public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers; this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon( this.defaultFavicon =
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif"); new Favicon(
} Resources.toByteArray(
Objects.requireNonNull(
getClass().getResource("/images/default_favicon.gif"))),
"image/gif");
}
public Favicon fetchFavicon(Feed feed) { public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) { for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed); Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) { if (isFaviconValid(icon)) {
return icon; return icon;
} }
} }
return defaultFavicon; return defaultFavicon;
} }
private static boolean isFaviconValid(Favicon favicon) { private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) { if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false; return false;
} }
long length = favicon.icon().length; long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) { if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH); log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false; return false;
} }
if (length > MAX_ICON_LENGTH) { if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH); log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false; return false;
} }
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) { if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType()); log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false; return false;
} }
return true; return true;
} }
} }

54
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,46 +1,42 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.time.Instant;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests; import com.commafeed.backend.Digests;
import com.commafeed.backend.Urls; import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedDAO; import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models; import com.commafeed.backend.model.Models;
import jakarta.inject.Singleton;
import java.time.Instant;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@Singleton @Singleton
@RequiredArgsConstructor @RequiredArgsConstructor
public class FeedService { public class FeedService {
private final FeedDAO feedDAO; private final FeedDAO feedDAO;
public synchronized Feed findOrCreate(String url) { public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url); String normalizedUrl = Urls.normalize(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl); String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash); Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) { if (feed == null) {
feed = new Feed(); feed = new Feed();
feed.setUrl(url); feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl); feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash); feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT); feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed); feedDAO.persist(feed);
} }
return feed; return feed;
} }
public void update(Feed feed) {
String normalized = Urls.normalize(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public void update(Feed feed) {
String normalized = Urls.normalize(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
} }

193
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -1,12 +1,5 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Urls; import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.FeedEntryStatusDAO;
@@ -18,107 +11,125 @@ import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount; import com.commafeed.frontend.model.UnreadCount;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Singleton @Singleton
public class FeedSubscriptionService { public class FeedSubscriptionService {
private final FeedEntryStatusDAO feedEntryStatusDAO; private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO; private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService; private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine; private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO, FeedService feedService, public FeedSubscriptionService(
FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) { FeedEntryStatusDAO feedEntryStatusDAO,
this.feedEntryStatusDAO = feedEntryStatusDAO; FeedSubscriptionDAO feedSubscriptionDAO,
this.feedSubscriptionDAO = feedSubscriptionDAO; FeedService feedService,
this.feedService = feedService; FeedRefreshEngine feedRefreshEngine,
this.feedRefreshEngine = feedRefreshEngine; CommaFeedConfiguration config) {
this.config = config; this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to // automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine // we need to use this hook because the feed needs to have been persisted before being
feedSubscriptionDAO.onPostCommitInsert(sub -> { // processed
Feed feed = sub.getFeed(); // by the feed engine
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) { feedSubscriptionDAO.onPostCommitInsert(
feedRefreshEngine.refreshImmediately(feed); sub -> {
} Feed feed = sub.getFeed();
}); if (feed.getDisabledUntil() == null
} || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) { public long subscribe(
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser(); User user, String url, String title, FeedCategory category, int position) {
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) { Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)", if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
maxFeedsPerUser); String message =
throw new FeedSubscriptionException(message); String.format(
} "You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url); Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http // upgrade feed to https if it was using http
if (Urls.isHttp(feed.getUrl()) && Urls.isHttps(url)) { if (Urls.isHttp(feed.getUrl()) && Urls.isHttps(url)) {
feed.setUrl(url); feed.setUrl(url);
} }
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed); FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) { if (sub == null) {
sub = new FeedSubscription(); sub = new FeedSubscription();
sub.setFeed(feed); sub.setFeed(feed);
sub.setUser(user); sub.setUser(user);
} }
sub.setCategory(category); sub.setCategory(category);
sub.setPosition(position); sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128)); sub.setTitle(FeedUtils.truncate(title, 128));
return feedSubscriptionDAO.merge(sub).getId(); return feedSubscriptionDAO.merge(sub).getId();
} }
public boolean unsubscribe(User user, Long subId) { public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId); FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) { if (sub != null) {
feedSubscriptionDAO.delete(sub); feedSubscriptionDAO.delete(sub);
return true; return true;
} else { } else {
return false; return false;
} }
} }
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException { public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh(); Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) { if (lastForceRefresh != null
throw new ForceFeedRefreshTooSoonException(); && lastForceRefresh
} .plus(config.feedRefresh().forceRefreshCooldownDuration())
.isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user); List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) { for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed(); Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed); feedRefreshEngine.refreshImmediately(feed);
} }
user.setLastForceRefresh(Instant.now()); user.setLastForceRefresh(Instant.now());
} }
public Map<Long, UnreadCount> getUnreadCount(User user) { public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user) return feedSubscriptionDAO.findAll(user).stream()
.stream() .collect(
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount)); Collectors.toMap(
} FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial") @SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException { public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) { private FeedSubscriptionException(String msg) {
super(msg); super(msg);
} }
} }
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
} }

14
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java
View File

@@ -1,21 +1,19 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail; import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer; import io.quarkus.mailer.Mailer;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class MailService { public class MailService {
private final Mailer mailer; private final Mailer mailer;
public void sendMail(User user, String subject, String content) { public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content); Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail); mailer.send(mail);
} }
} }

127
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java
View File

@@ -1,20 +1,16 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.security.spec.KeySpec; import java.security.spec.KeySpec;
import javax.crypto.SecretKey; import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory; import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html // taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@Slf4j @Slf4j
@@ -22,72 +18,71 @@ import lombok.extern.slf4j.Slf4j;
@Singleton @Singleton
public class PasswordEncryptionService { public class PasswordEncryptionService {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) { public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) { if (StringUtils.isBlank(attemptedPassword)) {
return false; return false;
} }
// Encrypt the clear-text password using the same salt that was used to // Encrypt the clear-text password using the same salt that was used to
// encrypt the original password // encrypt the original password
byte[] encryptedAttemptedPassword = null; byte[] encryptedAttemptedPassword = null;
try { try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt); encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) { } catch (Exception e) {
// should never happen // should never happen
log.error(e.getMessage(), e); log.error(e.getMessage(), e);
} }
if (encryptedAttemptedPassword == null) { if (encryptedAttemptedPassword == null) {
return false; return false;
} }
// Authentication succeeds if encrypted password that the user entered // Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash // is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword); return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
} }
public byte[] getEncryptedPassword(String password, byte[] salt) { public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST // PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for // specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2 // PBKDF2
String algorithm = "PBKDF2WithHmacSHA1"; String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here // SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160; int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at // Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations: // least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf // http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000: // iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/ // http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000; int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength); KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null; byte[] bytes = null;
try { try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm); SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec); SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded(); bytes = key.getEncoded();
} catch (Exception e) { } catch (Exception e) {
// should never happen // should never happen
log.error(e.getMessage(), e); log.error(e.getMessage(), e);
} }
return bytes; return bytes;
} }
public byte[] generateSalt() { public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random // VERY important to use SecureRandom instead of just Random
byte[] salt = null; byte[] salt = null;
try { try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG"); SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
} }

307
commafeed-server/src/main/java/com/commafeed/backend/service/PushNotificationService.java
View File

@@ -1,12 +1,20 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpClientFactory;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings.PushNotificationUserSettings;
import io.vertx.core.json.JsonObject;
import jakarta.inject.Singleton;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import lombok.extern.slf4j.Slf4j;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.config.RequestConfig;
@@ -18,163 +26,196 @@ import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair; import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.util.Timeout; import org.apache.hc.core5.util.Timeout;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpClientFactory;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings.PushNotificationUserSettings;
import io.vertx.core.json.JsonObject;
import lombok.extern.slf4j.Slf4j;
@Singleton @Singleton
@Slf4j @Slf4j
public class PushNotificationService { public class PushNotificationService {
private final CloseableHttpClient httpClient; private final CloseableHttpClient httpClient;
private final Meter meter; private final Meter meter;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
public PushNotificationService(HttpClientFactory httpClientFactory, MetricRegistry metrics, CommaFeedConfiguration config) { public PushNotificationService(
this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads()); HttpClientFactory httpClientFactory,
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify")); MetricRegistry metrics,
this.config = config; CommaFeedConfiguration config) {
} this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads());
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify"));
this.config = config;
}
public void notify(PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) { public void notify(
if (!config.pushNotifications().enabled() || settings.getType() == null) { PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
return; if (!config.pushNotifications().enabled() || settings.getType() == null) {
} return;
}
log.debug("sending {} push notification for entry {} in feed {}", settings.getType(), entry.getId(), log.debug(
subscription.getFeed().getId()); "sending {} push notification for entry {} in feed {}",
String entryTitle = entry.getContent() != null ? entry.getContent().getTitle() : null; settings.getType(),
String entryUrl = entry.getUrl(); entry.getId(),
String feedTitle = subscription.getTitle(); subscription.getFeed().getId());
String entryTitle = entry.getContent() != null ? entry.getContent().getTitle() : null;
String entryUrl = entry.getUrl();
String feedTitle = subscription.getTitle();
if (StringUtils.isBlank(entryTitle)) { if (StringUtils.isBlank(entryTitle)) {
entryTitle = "New entry"; entryTitle = "New entry";
} }
try { try {
switch (settings.getType()) { switch (settings.getType()) {
case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl); case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl);
case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl); case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl);
case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl); case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl);
default -> throw new IllegalStateException("unsupported notification type: " + settings.getType()); default ->
} throw new IllegalStateException(
} catch (IOException e) { "unsupported notification type: " + settings.getType());
throw new PushNotificationException("Failed to send external notification", e); }
} } catch (IOException e) {
throw new PushNotificationException("Failed to send external notification", e);
}
meter.mark(); meter.mark();
} }
private void sendNtfy(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl) throws IOException { private void sendNtfy(
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl()); PushNotificationUserSettings settings,
String topic = settings.getTopic(); String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String topic = settings.getTopic();
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(topic)) { if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(topic)) {
log.warn("ntfy notification skipped: missing server URL or topic"); log.warn("ntfy notification skipped: missing server URL or topic");
return; return;
} }
HttpPost request = new HttpPost(serverUrl + "/" + topic); HttpPost request = new HttpPost(serverUrl + "/" + topic);
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build()); request.setConfig(
request.addHeader("Title", feedTitle); RequestConfig.custom()
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8)); .setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.addHeader("Title", feedTitle);
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(entryUrl)) { if (StringUtils.isNotBlank(entryUrl)) {
request.addHeader("Click", entryUrl); request.addHeader("Click", entryUrl);
} }
if (StringUtils.isNotBlank(settings.getUserSecret())) { if (StringUtils.isNotBlank(settings.getUserSecret())) {
request.addHeader("Authorization", "Bearer " + settings.getUserSecret()); request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
} }
httpClient.execute(request, response -> { httpClient.execute(
if (response.getCode() >= 400) { request,
throw new PushNotificationException("ntfy notification failed with status " + response.getCode()); response -> {
} if (response.getCode() >= 400) {
return null; throw new PushNotificationException(
}); "ntfy notification failed with status " + response.getCode());
} }
return null;
});
}
private void sendGotify(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl) private void sendGotify(
throws IOException { PushNotificationUserSettings settings,
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl()); String feedTitle,
String token = settings.getUserSecret(); String entryTitle,
String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String token = settings.getUserSecret();
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(token)) { if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(token)) {
log.warn("gotify notification skipped: missing server URL or token"); log.warn("gotify notification skipped: missing server URL or token");
return; return;
} }
JsonObject json = new JsonObject(); JsonObject json = new JsonObject();
json.put("title", feedTitle); json.put("title", feedTitle);
json.put("message", entryTitle); json.put("message", entryTitle);
json.put("priority", 5); json.put("priority", 5);
if (StringUtils.isNotBlank(entryUrl)) { if (StringUtils.isNotBlank(entryUrl)) {
json.put("extras", json.put(
new JsonObject().put("client::notification", new JsonObject().put("click", new JsonObject().put("url", entryUrl)))); "extras",
} new JsonObject()
.put(
"client::notification",
new JsonObject()
.put("click", new JsonObject().put("url", entryUrl))));
}
HttpPost request = new HttpPost(serverUrl + "/message"); HttpPost request = new HttpPost(serverUrl + "/message");
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build()); request.setConfig(
request.addHeader("X-Gotify-Key", token); RequestConfig.custom()
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON)); .setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.addHeader("X-Gotify-Key", token);
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON));
httpClient.execute(request, response -> { httpClient.execute(
if (response.getCode() >= 400) { request,
throw new PushNotificationException("gotify notification failed with status " + response.getCode()); response -> {
} if (response.getCode() >= 400) {
return null; throw new PushNotificationException(
}); "gotify notification failed with status " + response.getCode());
} }
return null;
});
}
private void sendPushover(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl) private void sendPushover(
throws IOException { PushNotificationUserSettings settings,
String token = settings.getUserSecret(); String feedTitle,
String userKey = settings.getUserId(); String entryTitle,
String entryUrl)
throws IOException {
String token = settings.getUserSecret();
String userKey = settings.getUserId();
if (StringUtils.isBlank(token) || StringUtils.isBlank(userKey)) { if (StringUtils.isBlank(token) || StringUtils.isBlank(userKey)) {
log.warn("pushover notification skipped: missing token or user key"); log.warn("pushover notification skipped: missing token or user key");
return; return;
} }
List<NameValuePair> params = new ArrayList<>(); List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("token", token)); params.add(new BasicNameValuePair("token", token));
params.add(new BasicNameValuePair("user", userKey)); params.add(new BasicNameValuePair("user", userKey));
params.add(new BasicNameValuePair("title", feedTitle)); params.add(new BasicNameValuePair("title", feedTitle));
params.add(new BasicNameValuePair("message", entryTitle)); params.add(new BasicNameValuePair("message", entryTitle));
if (StringUtils.isNotBlank(entryUrl)) { if (StringUtils.isNotBlank(entryUrl)) {
params.add(new BasicNameValuePair("url", entryUrl)); params.add(new BasicNameValuePair("url", entryUrl));
} }
HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json"); HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json");
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build()); request.setConfig(
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8)); RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
httpClient.execute(request, response -> { httpClient.execute(
if (response.getCode() >= 400) { request,
throw new PushNotificationException("pushover notification failed with status " + response.getCode()); response -> {
} if (response.getCode() >= 400) {
return null; throw new PushNotificationException(
}); "pushover notification failed with status " + response.getCode());
} }
return null;
});
}
public static class PushNotificationException extends RuntimeException { public static class PushNotificationException extends RuntimeException {
private static final long serialVersionUID = -3392881821584833819L; private static final long serialVersionUID = -3392881821584833819L;
public PushNotificationException(String message) { public PushNotificationException(String message) {
super(message); super(message);
} }
public PushNotificationException(String message, Throwable cause) { public PushNotificationException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
} }
} }

241
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.service; package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConstants; import com.commafeed.CommaFeedConstants;
import com.commafeed.backend.Digests; import com.commafeed.backend.Digests;
@@ -24,138 +13,152 @@ import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role; import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities; import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class UserService { public class UserService {
private final FeedCategoryDAO feedCategoryDAO; private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO; private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO; private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO; private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO; private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService; private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities; private final PostLoginActivities postLoginActivities;
/** /** try to log in with given credentials */
* try to log in with given credentials public Optional<User> login(String nameOrEmail, String password) {
*/ if (nameOrEmail == null || password == null) {
public Optional<User> login(String nameOrEmail, String password) { return Optional.empty();
if (nameOrEmail == null || password == null) { }
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail); User user = userDAO.findByName(nameOrEmail);
if (user == null) { if (user == null) {
user = userDAO.findByEmail(nameOrEmail); user = userDAO.findByEmail(nameOrEmail);
} }
if (user != null && !user.isDisabled()) { if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt()); boolean authenticated =
if (authenticated) { encryptionService.authenticate(password, user.getPassword(), user.getSalt());
performPostLoginActivities(user); if (authenticated) {
return Optional.of(user); performPostLoginActivities(user);
} return Optional.of(user);
} }
return Optional.empty(); }
} return Optional.empty();
}
/** /** try to log in with given api key */
* try to log in with given api key public Optional<User> login(String apiKey) {
*/ if (apiKey == null) {
public Optional<User> login(String apiKey) { return Optional.empty();
if (apiKey == null) { }
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey); User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) { if (user != null && !user.isDisabled()) {
performPostLoginActivities(user); performPostLoginActivities(user);
return Optional.of(user); return Optional.of(user);
} }
return Optional.empty(); return Optional.empty();
} }
/** /** try to log in with given fever api key */
* try to log in with given fever api key public Optional<User> login(long userId, String feverApiKey) {
*/ if (feverApiKey == null) {
public Optional<User> login(long userId, String feverApiKey) { return Optional.empty();
if (feverApiKey == null) { }
return Optional.empty();
}
User user = userDAO.findById(userId); User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) { if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty(); return Optional.empty();
} }
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey()); String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equalsIgnoreCase(feverApiKey)) { if (!computedFeverApiKey.equalsIgnoreCase(feverApiKey)) {
return Optional.empty(); return Optional.empty();
} }
performPostLoginActivities(user); performPostLoginActivities(user);
return Optional.of(user); return Optional.of(user);
} }
/** /** should triggers after successful login */
* should triggers after successful login public void performPostLoginActivities(User user) {
*/ postLoginActivities.executeFor(user);
public void performPostLoginActivities(User user) { }
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) { public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false); return register(name, password, email, roles, false);
} }
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) { public User register(
String name,
String password,
String email,
Collection<Role> roles,
boolean forceRegistration) {
if (!forceRegistration) { if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance"); Preconditions.checkState(
} config.users().allowRegistrations(),
"Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken"); Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) { if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken"); Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
} }
User user = new User(); User user = new User();
byte[] salt = encryptionService.generateSalt(); byte[] salt = encryptionService.generateSalt();
user.setName(name); user.setName(name);
user.setEmail(email); user.setEmail(email);
user.setCreated(Instant.now()); user.setCreated(Instant.now());
user.setSalt(salt); user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt)); user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.persist(user); userDAO.persist(user);
for (Role role : roles) { for (Role role : roles) {
userRoleDAO.persist(new UserRole(user, role)); userRoleDAO.persist(new UserRole(user, role));
} }
return user; return user;
} }
public void createDemoUser() { public void createDemoUser() {
register(CommaFeedConstants.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true); register(
} CommaFeedConstants.USERNAME_DEMO,
"demo",
"demo@commafeed.com",
Collections.singletonList(Role.USER),
true);
}
public void unregister(User user) { public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user)); userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user)); userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user)); feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user)); feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user); userDAO.delete(user);
} }
public String generateApiKey(User user) { public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt()); byte[] key =
return Digests.sha1Hex(key); encryptionService.getEncryptedPassword(
} UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) { public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user); return userRoleDAO.findRoles(user);
} }
} }

328
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -1,10 +1,5 @@
package com.commafeed.backend.service.db; package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
@@ -16,173 +11,202 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel; import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
/** /** Contains utility methods for cleaning the database */
* Contains utility methods for cleaning the database
*
*/
@Slf4j @Slf4j
@Singleton @Singleton
public class DatabaseCleaningService { public class DatabaseCleaningService {
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO; private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO; private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO; private final FeedEntryStatusDAO feedEntryStatusDAO;
private final int batchSize; private final int batchSize;
private final boolean keepStarredEntries; private final boolean keepStarredEntries;
private final Meter entriesDeletedMeter; private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO, public DatabaseCleaningService(
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) { CommaFeedConfiguration config,
this.unitOfWork = unitOfWork; UnitOfWork unitOfWork,
this.feedDAO = feedDAO; FeedDAO feedDAO,
this.feedEntryDAO = feedEntryDAO; FeedEntryDAO feedEntryDAO,
this.feedEntryContentDAO = feedEntryContentDAO; FeedEntryContentDAO feedEntryContentDAO,
this.feedEntryStatusDAO = feedEntryStatusDAO; FeedEntryStatusDAO feedEntryStatusDAO,
this.batchSize = config.database().cleanup().batchSize(); MetricRegistry metrics) {
this.keepStarredEntries = config.database().cleanup().keepStarredEntries(); this.unitOfWork = unitOfWork;
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted")); this.feedDAO = feedDAO;
} this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.keepStarredEntries = config.database().cleanup().keepStarredEntries();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() { public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions"); log.info("cleaning feeds without subscriptions");
long total = 0; long total = 0;
int deleted; int deleted;
long entriesTotal = 0; long entriesTotal = 0;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions"); log.info("interrupted, stopping cleanup of feeds without subscriptions");
return; return;
} }
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1)); List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) { for (Feed feed : feeds) {
long entriesDeleted; long entriesDeleted;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions"); log.info("interrupted, stopping cleanup of feeds without subscriptions");
return; return;
} }
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize)); entriesDeleted =
entriesDeletedMeter.mark(entriesDeleted); unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesTotal += entriesDeleted; entriesDeletedMeter.mark(entriesDeleted);
log.debug("removed {} entries for feeds without subscriptions", entriesTotal); entriesTotal += entriesDeleted;
} while (entriesDeleted > 0); log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} } while (entriesDeleted > 0);
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList()))); }
total += deleted; deleted =
log.debug("removed {} feeds without subscriptions", total); unitOfWork.call(
} while (deleted != 0); () ->
log.info("cleanup done: {} feeds without subscriptions deleted", total); feedDAO.delete(
} feedDAO.findByIds(
feeds.stream()
.map(AbstractModel::getId)
.toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() { public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries"); log.info("cleaning contents without entries");
long total = 0; long total = 0;
long deleted; long deleted;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of contents without entries"); log.info("interrupted, stopping cleanup of contents without entries");
return; return;
} }
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize)); deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted; total += deleted;
log.debug("removed {} contents without entries", total); log.debug("removed {} contents without entries", total);
} while (deleted != 0); } while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total); log.info("cleanup done: {} contents without entries deleted", total);
} }
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity"); log.info("cleaning entries exceeding feed capacity");
long total = 0; long total = 0;
while (true) { while (true) {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity"); log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return; return;
} }
List<FeedCapacity> feeds = unitOfWork List<FeedCapacity> feeds =
.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize, keepStarredEntries)); unitOfWork.call(
if (feeds.isEmpty()) { () ->
break; feedEntryDAO.findFeedsExceedingCapacity(
} maxFeedCapacity, batchSize, keepStarredEntries));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) { for (final FeedCapacity feed : feeds) {
long remaining = feed.capacity() - maxFeedCapacity; long remaining = feed.capacity() - maxFeedCapacity;
int deleted; int deleted;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity"); log.info(
return; "interrupted, stopping cleanup of entries exceeding feed capacity");
} return;
}
final long rem = remaining; final long rem = remaining;
deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.id(), Math.min(batchSize, rem), keepStarredEntries)); deleted =
entriesDeletedMeter.mark(deleted); unitOfWork.call(
total += deleted; () ->
remaining -= deleted; feedEntryDAO.deleteOldEntries(
log.debug("removed {} entries for feeds exceeding capacity", total); feed.id(),
} while (deleted > 0 && remaining > 0); Math.min(batchSize, rem),
} keepStarredEntries));
} entriesDeletedMeter.mark(deleted);
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total); total += deleted;
} remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (deleted > 0 && remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) { public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries"); log.info("cleaning old entries");
long total = 0; long total = 0;
long deleted; long deleted;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old entries"); log.info("interrupted, stopping cleanup of old entries");
return; return;
} }
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize, keepStarredEntries)); deleted =
entriesDeletedMeter.mark(deleted); unitOfWork.call(
total += deleted; () ->
log.debug("removed {} old entries", total); feedEntryDAO.deleteEntriesOlderThan(
} while (deleted != 0); olderThan, batchSize, keepStarredEntries));
log.info("cleanup done: {} old entries deleted", total); entriesDeletedMeter.mark(deleted);
} total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) { public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses"); log.info("cleaning old read statuses");
long total = 0; long total = 0;
long deleted; long deleted;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old read statuses"); log.info("interrupted, stopping cleanup of old read statuses");
return; return;
} }
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize)); deleted =
total += deleted; unitOfWork.call(
log.debug("removed {} old read statuses", total); () -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
} while (deleted != 0); total += deleted;
log.info("cleanup done: {} old read statuses deleted", total); log.debug("removed {} old read statuses", total);
} } while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
public void autoMarkAsRead() { public void autoMarkAsRead() {
log.info("marking entries as read based on autoMarkAsReadAfterDays"); log.info("marking entries as read based on autoMarkAsReadAfterDays");
long total = 0; long total = 0;
long marked; long marked;
do { do {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping marking entries as read"); log.info("interrupted, stopping marking entries as read");
return; return;
} }
marked = unitOfWork.call(() -> feedEntryStatusDAO.autoMarkAsRead(batchSize)); marked = unitOfWork.call(() -> feedEntryStatusDAO.autoMarkAsRead(batchSize));
total += marked; total += marked;
log.debug("marked {} entries as read", total); log.debug("marked {} entries as read", total);
} while (marked != 0); } while (marked != 0);
log.info("cleanup done: marked {} entries as read", total); log.info("cleanup done: marked {} entries as read", total);
} }
} }

48
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -1,44 +1,42 @@
package com.commafeed.backend.service.db; package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO; import com.commafeed.backend.dao.UserDAO;
import jakarta.inject.Singleton;
import liquibase.database.Database; import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase; import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject; import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.kohsuke.MetaInfServices;
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class DatabaseStartupService { public class DatabaseStartupService {
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final UserDAO userDAO; private final UserDAO userDAO;
public boolean isInitialSetupRequired() { public boolean isInitialSetupRequired() {
return unitOfWork.call(userDAO::count) == 0; return unitOfWork.call(userDAO::count) == 0;
} }
/** /**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns * Register a postgresql database in liquibase that doesn't escape columns, so that we can use
*/ * lower case columns
@MetaInfServices(Database.class) */
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase { @MetaInfServices(Database.class)
@Override public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) { @Override
return objectName; public String escapeObjectName(
} String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
@Override }
public int getPriority() {
return super.getPriority() + 1;
}
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
} }

33
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java
View File

@@ -1,30 +1,29 @@
package com.commafeed.backend.service.internal; package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO; import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class PostLoginActivities { public class PostLoginActivities {
private final UserDAO userDAO; private final UserDAO userDAO;
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
public void executeFor(User user) { public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in // only update lastLogin every once in a while in order to avoid invalidating the cache
Instant now = Instant.now(); // every
Instant lastLogin = user.getLastLogin(); // time someone logs in
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) { Instant now = Instant.now();
user.setLastLogin(now); Instant lastLogin = user.getLastLogin();
unitOfWork.run(() -> userDAO.merge(user)); if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
} user.setLastLogin(now);
} unitOfWork.run(() -> userDAO.merge(user));
}
}
} }

42
commafeed-server/src/main/java/com/commafeed/backend/task/AutoMarkAsReadTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService; import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class AutoMarkAsReadTask extends ScheduledTask { public class AutoMarkAsReadTask extends ScheduledTask {
private final DatabaseCleaningService cleaner; private final DatabaseCleaningService cleaner;
@Override @Override
public void run() { public void run() {
cleaner.autoMarkAsRead(); cleaner.autoMarkAsRead();
} }
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 30; return 30;
} }
@Override @Override
public long getPeriod() { public long getPeriod() {
return 60; return 60;
} }
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

72
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java
View File

@@ -1,16 +1,13 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConstants; import com.commafeed.CommaFeedConstants;
import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO; import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User; import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService; import com.commafeed.backend.service.UserService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -19,43 +16,42 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class DemoAccountCleanupTask extends ScheduledTask { public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork; private final UnitOfWork unitOfWork;
private final UserDAO userDAO; private final UserDAO userDAO;
private final UserService userService; private final UserService userService;
@Override @Override
protected void run() { protected void run() {
if (!config.users().createDemoAccount()) { if (!config.users().createDemoAccount()) {
return; return;
} }
log.info("recreating demo user account"); log.info("recreating demo user account");
unitOfWork.run(() -> { unitOfWork.run(
User demoUser = userDAO.findByName(CommaFeedConstants.USERNAME_DEMO); () -> {
if (demoUser == null) { User demoUser = userDAO.findByName(CommaFeedConstants.USERNAME_DEMO);
return; if (demoUser == null) {
} return;
}
userService.unregister(demoUser); userService.unregister(demoUser);
userService.createDemoUser(); userService.createDemoUser();
}); });
}
} @Override
protected long getInitialDelay() {
return 1;
}
@Override @Override
protected long getInitialDelay() { protected long getPeriod() {
return 1; return getTimeUnit().convert(24, TimeUnit.HOURS);
} }
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

50
commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java
View File

@@ -1,42 +1,38 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService; import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask { public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner; private final DatabaseCleaningService cleaner;
@Override @Override
public void run() { public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity(); int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) { if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity); cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
} }
} }
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 10; return 10;
} }
@Override @Override
public long getPeriod() { public long getPeriod() {
return 60; return 60;
} }
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

56
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java
View File

@@ -1,45 +1,41 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService; import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class OldEntriesCleanupTask extends ScheduledTask { public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner; private final DatabaseCleaningService cleaner;
@Override @Override
public void run() { public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge(); Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) { if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge); Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold); cleaner.cleanEntriesOlderThan(threshold);
} }
} }
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 5; return 5;
} }
@Override @Override
public long getPeriod() { public long getPeriod() {
return 60; return 60;
} }
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

52
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java
View File

@@ -1,43 +1,39 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService; import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class OldStatusesCleanupTask extends ScheduledTask { public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner; private final DatabaseCleaningService cleaner;
@Override @Override
public void run() { public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold(); Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) { if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold); cleaner.cleanStatusesOlderThan(threshold);
} }
} }
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 15; return 15;
} }
@Override @Override
public long getPeriod() { public long getPeriod() {
return 60; return 60;
} }
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

42
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService; import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask { public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner; private final DatabaseCleaningService cleaner;
@Override @Override
public void run() { public void run() {
cleaner.cleanContentsWithoutEntries(); cleaner.cleanContentsWithoutEntries();
} }
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 25; return 25;
} }
@Override @Override
public long getPeriod() { public long getPeriod() {
return 60; return 60;
} }
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

42
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService; import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
@Singleton @Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask { public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner; private final DatabaseCleaningService cleaner;
@Override @Override
public void run() { public void run() {
cleaner.cleanFeedsWithoutSubscriptions(); cleaner.cleanFeedsWithoutSubscriptions();
} }
@Override @Override
public long getInitialDelay() { public long getInitialDelay() {
return 20; return 20;
} }
@Override @Override
public long getPeriod() { public long getPeriod() {
return 60; return 60;
} }
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
} }

39
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java
View File

@@ -2,29 +2,34 @@ package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public abstract class ScheduledTask { public abstract class ScheduledTask {
protected abstract void run(); protected abstract void run();
protected abstract long getInitialDelay(); protected abstract long getInitialDelay();
protected abstract long getPeriod(); protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit(); protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) { public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> { Runnable runnable =
try { () -> {
ScheduledTask.this.run(); try {
} catch (Exception e) { ScheduledTask.this.run();
log.error(e.getMessage(), e); } catch (Exception e) {
} log.error(e.getMessage(), e);
}; }
log.debug("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), };
getTimeUnit(), getInitialDelay(), getTimeUnit()); log.debug(
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit()); "registering task {} for execution every {} {}, starting in {} {}",
} getClass().getSimpleName(),
getPeriod(),
getTimeUnit(),
getInitialDelay(),
getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
} }

39
commafeed-server/src/main/java/com/commafeed/backend/task/TaskScheduler.java
View File

@@ -1,37 +1,34 @@
package com.commafeed.backend.task; package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import io.quarkus.arc.All; import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Singleton @Singleton
public class TaskScheduler { public class TaskScheduler {
private final List<ScheduledTask> tasks; private final List<ScheduledTask> tasks;
private final CommaFeedConfiguration config; private final CommaFeedConfiguration config;
private ScheduledExecutorService executor; private ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks, CommaFeedConfiguration config) { public TaskScheduler(@All List<ScheduledTask> tasks, CommaFeedConfiguration config) {
this.tasks = tasks; this.tasks = tasks;
this.config = config; this.config = config;
} }
public void start() { public void start() {
this.executor = Executors.newScheduledThreadPool(tasks.size()); this.executor = Executors.newScheduledThreadPool(tasks.size());
this.tasks.forEach(task -> task.register(executor)); this.tasks.forEach(task -> task.register(executor));
} }
public void stop() { public void stop() {
MoreExecutors.shutdownAndAwaitTermination(executor, config.shutdownTimeout()); MoreExecutors.shutdownAndAwaitTermination(executor, config.shutdownTimeout());
} }
} }

7
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java
View File

@@ -2,11 +2,8 @@ package com.commafeed.backend.urlprovider;
import java.util.List; import java.util.List;
/** /** Tries to find a feed url given the url and page content */
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider { public interface FeedURLProvider {
List<String> get(String url, String urlContent); List<String> get(String url, String urlContent);
} }

27
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java
View File

@@ -1,25 +1,24 @@
package com.commafeed.backend.urlprovider; package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import java.util.List; import java.util.List;
import java.util.stream.Stream; import java.util.stream.Stream;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
@Singleton @Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider { public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override @Override
public List<String> get(String url, String urlContent) { public List<String> get(String url, String urlContent) {
Document doc = Jsoup.parse(urlContent, url); Document doc = Jsoup.parse(urlContent, url);
if (!"html".equals(doc.children().getFirst().tagName())) { if (!"html".equals(doc.children().getFirst().tagName())) {
return List.of(); return List.of();
} }
return Stream.concat(doc.select("link[type=application/atom+xml]").stream(), doc.select("link[type=application/rss+xml]").stream()) return Stream.concat(
.map(node -> node.attr("abs:href")) doc.select("link[type=application/atom+xml]").stream(),
.toList(); doc.select("link[type=application/rss+xml]").stream())
} .map(node -> node.attr("abs:href"))
.toList();
}
} }

28
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java
View File

@@ -1,30 +1,28 @@
package com.commafeed.backend.urlprovider; package com.commafeed.backend.urlprovider;
import java.util.List;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
import java.util.List;
import org.apache.commons.lang3.Strings; import org.apache.commons.lang3.Strings;
/** /**
* Workaround for Youtube channels * Workaround for Youtube channels
* *
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL * <p>converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID * https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/ */
@Singleton @Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider { public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/"; private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String REPLACEMENT_PREFIX =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override @Override
public List<String> get(String url, String urlContent) { public List<String> get(String url, String urlContent) {
if (!Strings.CI.startsWith(url, PREFIX)) { if (!Strings.CI.startsWith(url, PREFIX)) {
return List.of(); return List.of();
} }
return List.of(REPLACEMENT_PREFIX + url.substring(PREFIX.length()));
}
return List.of(REPLACEMENT_PREFIX + url.substring(PREFIX.length()));
}
} }

40
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java
View File

@@ -1,13 +1,11 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data; import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "Entry details") @Schema(description = "Entry details")
@@ -15,27 +13,27 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class Category implements Serializable { public class Category implements Serializable {
@Schema(description = "category id", required = true) @Schema(description = "category id", required = true)
private String id; private String id;
@Schema(description = "parent category id") @Schema(description = "parent category id")
private String parentId; private String parentId;
@Schema(description = "parent category name") @Schema(description = "parent category name")
private String parentName; private String parentName;
@Schema(description = "category id", required = true) @Schema(description = "category id", required = true)
private String name; private String name;
@Schema(description = "category children categories", required = true) @Schema(description = "category children categories", required = true)
private List<Category> children = new ArrayList<>(); private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", required = true) @Schema(description = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>(); private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", required = true) @Schema(description = "whether the category is expanded or collapsed", required = true)
private boolean expanded; private boolean expanded;
@Schema(description = "position of the category in the list", required = true) @Schema(description = "position of the category in the list", required = true)
private int position; private int position;
} }

54
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java
View File

@@ -1,13 +1,11 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data; import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata") @Schema(description = "List of entries with some metadata")
@@ -15,36 +13,38 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class Entries implements Serializable { public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", required = true) @Schema(description = "name of the feed or the category requested", required = true)
private String name; private String name;
@Schema(description = "error or warning message") @Schema(description = "error or warning message")
private String message; private String message;
@Schema(description = "times the server tried to refresh the feed and failed", required = true) @Schema(description = "times the server tried to refresh the feed and failed", required = true)
private int errorCount; private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries") @Schema(
private String feedLink; description =
"URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", required = true) @Schema(description = "list generation timestamp", required = true)
private long timestamp; private long timestamp;
@Schema(description = "if the query has more elements", required = true) @Schema(description = "if the query has more elements", required = true)
private boolean hasMore; private boolean hasMore;
@Schema(description = "the requested offset") @Schema(description = "the requested offset")
private int offset; private int offset;
@Schema(description = "the requested limit") @Schema(description = "the requested limit")
private int limit; private int limit;
@Schema(description = "list of entries", required = true) @Schema(description = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>(); private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
@Schema(
description =
"if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
} }

200
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java
View File

@@ -1,22 +1,19 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.Strings;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent; import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus; import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag; import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import lombok.Data; import lombok.Data;
import org.apache.commons.lang3.Strings;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "Entry details") @Schema(description = "Entry details")
@@ -24,123 +21,134 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class Entry implements Serializable { public class Entry implements Serializable {
@Schema(description = "entry id", required = true) @Schema(description = "entry id", required = true)
private String id; private String id;
@Schema(description = "entry guid", required = true) @Schema(description = "entry guid", required = true)
private String guid; private String guid;
@Schema(description = "entry title", required = true) @Schema(description = "entry title", required = true)
private String title; private String title;
@Schema(description = "entry content", required = true) @Schema(description = "entry content", required = true)
private String content; private String content;
@Schema(description = "comma-separated list of categories") @Schema(description = "comma-separated list of categories")
private String categories; private String categories;
@Schema(description = "whether entry content and title are rtl", required = true) @Schema(description = "whether entry content and title are rtl", required = true)
private boolean rtl; private boolean rtl;
@Schema(description = "entry author") @Schema(description = "entry author")
private String author; private String author;
@Schema(description = "entry enclosure url, if any") @Schema(description = "entry enclosure url, if any")
private String enclosureUrl; private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any") @Schema(description = "entry enclosure mime type, if any")
private String enclosureType; private String enclosureType;
@Schema(description = "entry media description, if any") @Schema(description = "entry media description, if any")
private String mediaDescription; private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any") @Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl; private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any") @Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth; private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any") @Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight; private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = SchemaType.INTEGER, required = true) @Schema(description = "entry publication date", type = SchemaType.INTEGER, required = true)
private Instant date; private Instant date;
@Schema(description = "entry insertion date in the database", type = SchemaType.INTEGER, required = true) @Schema(
private Instant insertedDate; description = "entry insertion date in the database",
type = SchemaType.INTEGER,
required = true)
private Instant insertedDate;
@Schema(description = "feed id", required = true) @Schema(description = "feed id", required = true)
private String feedId; private String feedId;
@Schema(description = "feed name", required = true) @Schema(description = "feed name", required = true)
private String feedName; private String feedName;
@Schema(description = "this entry's feed url", required = true) @Schema(description = "this entry's feed url", required = true)
private String feedUrl; private String feedUrl;
@Schema(description = "this entry's website url", required = true) @Schema(description = "this entry's website url", required = true)
private String feedLink; private String feedLink;
@Schema(description = "The favicon url to use for this feed", required = true) @Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl; private String iconUrl;
@Schema(description = "entry url", required = true) @Schema(description = "entry url", required = true)
private String url; private String url;
@Schema(description = "read status", required = true) @Schema(description = "read status", required = true)
private boolean read; private boolean read;
@Schema(description = "starred status", required = true) @Schema(description = "starred status", required = true)
private boolean starred; private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", required = true) @Schema(
private boolean markable; description = "whether the entry is still markable (old entry statuses are discarded)",
required = true)
private boolean markable;
@Schema(description = "tags", required = true) @Schema(description = "tags", required = true)
private List<String> tags; private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) { public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry(); Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry(); FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription(); FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent(); FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId())); entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid()); entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead()); entry.setRead(status.isRead());
entry.setStarred(status.isStarred()); entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable()); entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished()); entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted()); entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl()); entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle()); entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId())); entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl()); entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink()); entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub)); entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList()); entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) { if (content != null) {
entry.setRtl(content.getDirection() == FeedEntryContent.Direction.RTL); entry.setRtl(content.getDirection() == FeedEntryContent.Direction.RTL);
entry.setTitle(content.getTitle()); entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent()); entry.setContent(
entry.setAuthor(content.getAuthor()); proxyImages
? FeedUtils.proxyImages(content.getContent())
: content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType()); entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && Strings.CS.contains(content.getEnclosureType(), "image") entry.setEnclosureUrl(
? FeedUtils.proxyImage(content.getEnclosureUrl()) proxyImages && Strings.CS.contains(content.getEnclosureType(), "image")
: content.getEnclosureUrl()); ? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription()); entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl()); entry.setMediaThumbnailUrl(
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth()); proxyImages
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight()); ? FeedUtils.proxyImage(content.getMediaThumbnailUrl())
: content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories()); entry.setCategories(content.getCategories());
} }
return entry;
}
return entry;
}
} }

15
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java
View File

@@ -1,11 +1,9 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data; import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "Feed details") @Schema(description = "Feed details")
@@ -13,10 +11,9 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class FeedInfo implements Serializable { public class FeedInfo implements Serializable {
@Schema(description = "url", required = true) @Schema(description = "url", required = true)
private String url; private String url;
@Schema(description = "title", required = true)
private String title;
@Schema(description = "title", required = true)
private String title;
} }

62
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java
View File

@@ -1,11 +1,9 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data; import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "Server infos") @Schema(description = "Server infos")
@@ -13,46 +11,44 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class ServerInfo implements Serializable { public class ServerInfo implements Serializable {
@Schema @Schema private String announcement;
private String announcement;
@Schema(required = true) @Schema(required = true)
private String version; private String version;
@Schema(required = true) @Schema(required = true)
private String gitCommit; private String gitCommit;
@Schema(required = true) @Schema(required = true)
private boolean allowRegistrations; private boolean allowRegistrations;
@Schema(required = true) @Schema(required = true)
private boolean emailAddressRequired; private boolean emailAddressRequired;
@Schema(required = true) @Schema(required = true)
private boolean smtpEnabled; private boolean smtpEnabled;
@Schema(required = true) @Schema(required = true)
private boolean demoAccountEnabled; private boolean demoAccountEnabled;
@Schema(required = true) @Schema(required = true)
private boolean websocketEnabled; private boolean websocketEnabled;
@Schema(required = true) @Schema(required = true)
private long websocketPingInterval; private long websocketPingInterval;
@Schema(required = true) @Schema(required = true)
private long treeReloadInterval; private long treeReloadInterval;
@Schema(required = true) @Schema(required = true)
private long forceRefreshCooldownDuration; private long forceRefreshCooldownDuration;
@Schema(required = true) @Schema(required = true)
private boolean initialSetupRequired; private boolean initialSetupRequired;
@Schema(required = true) @Schema(required = true)
private int minimumPasswordLength; private int minimumPasswordLength;
@Schema(required = true)
private boolean pushNotificationsEnabled;
@Schema(required = true)
private boolean pushNotificationsEnabled;
} }

174
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.model.UserSettings.IconDisplayMode; import com.commafeed.backend.model.UserSettings.IconDisplayMode;
import com.commafeed.backend.model.UserSettings.PushNotificationType; import com.commafeed.backend.model.UserSettings.PushNotificationType;
import com.commafeed.backend.model.UserSettings.ReadingMode; import com.commafeed.backend.model.UserSettings.ReadingMode;
import com.commafeed.backend.model.UserSettings.ReadingOrder; import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.backend.model.UserSettings.ScrollMode; import com.commafeed.backend.model.UserSettings.ScrollMode;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data; import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "User settings") @Schema(description = "User settings")
@@ -19,113 +16,128 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class Settings implements Serializable { public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none") @Schema(description = "user's preferred language, english if none")
private String language; private String language;
@Schema(description = "user reads all entries or unread entries only", required = true) @Schema(description = "user reads all entries or unread entries only", required = true)
private ReadingMode readingMode; private ReadingMode readingMode;
@Schema(description = "user reads entries in ascending or descending order", required = true) @Schema(description = "user reads entries in ascending or descending order", required = true)
private ReadingOrder readingOrder; private ReadingOrder readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", required = true) @Schema(
private boolean showRead; description = "user wants category and feeds with no unread entries shown",
required = true)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", required = true) @Schema(
private boolean scrollMarks; description = "In expanded view, scroll through entries mark them as read",
required = true)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website") @Schema(description = "user's custom css for the website")
private String customCss; private String customCss;
@Schema(description = "user's custom js for the website") @Schema(description = "user's custom js for the website")
private String customJs; private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", required = true) @Schema(
private int scrollSpeed; description = "user's preferred scroll speed when navigating between entries",
required = true)
private int scrollSpeed;
@Schema(description = "whether to scroll to the selected entry", required = true) @Schema(description = "whether to scroll to the selected entry", required = true)
private ScrollMode scrollMode; private ScrollMode scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", required = true) @Schema(
private int entriesToKeepOnTopWhenScrolling; description = "number of entries to keep above the selected entry when scrolling",
required = true)
private int entriesToKeepOnTopWhenScrolling;
@Schema(description = "whether to show the star icon in the header of entries", required = true) @Schema(description = "whether to show the star icon in the header of entries", required = true)
private IconDisplayMode starIconDisplayMode; private IconDisplayMode starIconDisplayMode;
@Schema(description = "whether to show the external link icon in the header of entries", required = true) @Schema(
private IconDisplayMode externalLinkIconDisplayMode; description = "whether to show the external link icon in the header of entries",
required = true)
private IconDisplayMode externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", required = true) @Schema(description = "ask for confirmation when marking all entries as read", required = true)
private boolean markAllAsReadConfirmation; private boolean markAllAsReadConfirmation;
@Schema(description = "navigate to the next unread category or feed after marking all entries as read", required = true) @Schema(
private boolean markAllAsReadNavigateToNextUnread; description =
"navigate to the next unread category or feed after marking all entries as read",
required = true)
private boolean markAllAsReadNavigateToNextUnread;
@Schema(description = "show commafeed's own context menu on right click", required = true) @Schema(description = "show commafeed's own context menu on right click", required = true)
private boolean customContextMenu; private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", required = true) @Schema(
private boolean mobileFooter; description = "on mobile, show action buttons at the bottom of the screen",
required = true)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", required = true) @Schema(description = "show unread count in the title", required = true)
private boolean unreadCountTitle; private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", required = true) @Schema(description = "show unread count in the favicon", required = true)
private boolean unreadCountFavicon; private boolean unreadCountFavicon;
@Schema(description = "disable pull to refresh", required = true) @Schema(description = "disable pull to refresh", required = true)
private boolean disablePullToRefresh; private boolean disablePullToRefresh;
@Schema(description = "primary theme color to use in the UI") @Schema(description = "primary theme color to use in the UI")
private String primaryColor; private String primaryColor;
@Schema(description = "sharing settings", required = true) @Schema(description = "sharing settings", required = true)
private SharingSettings sharingSettings = new SharingSettings(); private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "push notification settings", required = true) @Schema(description = "push notification settings", required = true)
private PushNotificationSettings pushNotificationSettings = new PushNotificationSettings(); private PushNotificationSettings pushNotificationSettings = new PushNotificationSettings();
@Schema(description = "User notification settings") @Schema(description = "User notification settings")
@Data @Data
public static class PushNotificationSettings implements Serializable { public static class PushNotificationSettings implements Serializable {
@Schema(description = "notification provider type") @Schema(description = "notification provider type")
private PushNotificationType type; private PushNotificationType type;
@Schema(description = "server URL for ntfy or gotify") @Schema(description = "server URL for ntfy or gotify")
private String serverUrl; private String serverUrl;
@Schema(description = "user Id") @Schema(description = "user Id")
private String userId; private String userId;
@Schema(description = "user secret for authentication with the service") @Schema(description = "user secret for authentication with the service")
private String userSecret; private String userSecret;
@Schema(description = "topic") @Schema(description = "topic")
private String topic; private String topic;
} }
@Schema(description = "User sharing settings") @Schema(description = "User sharing settings")
@Data @Data
public static class SharingSettings implements Serializable { public static class SharingSettings implements Serializable {
@Schema(required = true) @Schema(required = true)
private boolean email; private boolean email;
@Schema(required = true) @Schema(required = true)
private boolean gmail; private boolean gmail;
@Schema(required = true) @Schema(required = true)
private boolean facebook; private boolean facebook;
@Schema(required = true) @Schema(required = true)
private boolean twitter; private boolean twitter;
@Schema(required = true) @Schema(required = true)
private boolean tumblr; private boolean tumblr;
@Schema(required = true) @Schema(required = true)
private boolean instapaper; private boolean instapaper;
@Schema(required = true) @Schema(required = true)
private boolean buffer; private boolean buffer;
} }
} }

141
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java
View File

@@ -1,18 +1,15 @@
package com.commafeed.frontend.model; package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory; import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection; import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import lombok.Data; import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial") @SuppressWarnings("serial")
@Schema(description = "User information") @Schema(description = "User information")
@@ -20,80 +17,90 @@ import lombok.Data;
@RegisterForReflection @RegisterForReflection
public class Subscription implements Serializable { public class Subscription implements Serializable {
@Schema(description = "subscription id", required = true) @Schema(description = "subscription id", required = true)
private Long id; private Long id;
@Schema(description = "subscription name", required = true) @Schema(description = "subscription name", required = true)
private String name; private String name;
@Schema(description = "error message while fetching the feed") @Schema(description = "error message while fetching the feed")
private String message; private String message;
@Schema(description = "error count", required = true) @Schema(description = "error count", required = true)
private int errorCount; private int errorCount;
@Schema(description = "last time the feed was refreshed", type = SchemaType.INTEGER) @Schema(description = "last time the feed was refreshed", type = SchemaType.INTEGER)
private Instant lastRefresh; private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = SchemaType.INTEGER) @Schema(
private Instant nextRefresh; description =
"next time the feed refresh is planned, null if refresh is already queued",
type = SchemaType.INTEGER)
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", required = true) @Schema(description = "this subscription's feed url", required = true)
private String feedUrl; private String feedUrl;
@Schema(description = "this subscription's website url", required = true) @Schema(description = "this subscription's website url", required = true)
private String feedLink; private String feedLink;
@Schema(description = "The favicon url to use for this feed", required = true) @Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl; private String iconUrl;
@Schema(description = "unread count", required = true) @Schema(description = "unread count", required = true)
private long unread; private long unread;
@Schema(description = "category id") @Schema(description = "category id")
private String categoryId; private String categoryId;
@Schema(description = "position of the subscription's in the list") @Schema(description = "position of the subscription's in the list")
private int position; private int position;
@Schema(description = "date of the newest item", type = SchemaType.INTEGER) @Schema(description = "date of the newest item", type = SchemaType.INTEGER)
private Instant newestItemTime; private Instant newestItemTime;
@Schema(description = "CEL string evaluated on new entries to mark them as read if they do not match") @Schema(
private String filter; description =
"CEL string evaluated on new entries to mark them as read if they do not match")
private String filter;
@Schema(description = "JEXL legacy filter") @Schema(description = "JEXL legacy filter")
private String filterLegacy; private String filterLegacy;
@Schema(description = "whether to send push notifications for new entries of this feed", required = true) @Schema(
private boolean pushNotificationsEnabled; description = "whether to send push notifications for new entries of this feed",
required = true)
private boolean pushNotificationsEnabled;
@Schema(description = "automatically mark entries as read after this many days (null to disable)") @Schema(
private Integer autoMarkAsReadAfterDays; description =
"automatically mark entries as read after this many days (null to disable)")
private Integer autoMarkAsReadAfterDays;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) { public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory(); FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed(); Feed feed = subscription.getFeed();
Subscription sub = new Subscription(); Subscription sub = new Subscription();
sub.setId(subscription.getId()); sub.setId(subscription.getId());
sub.setName(subscription.getTitle()); sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition()); sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage()); sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount()); sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl()); sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink()); sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription)); sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated()); sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh( sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil()); (feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now()))
sub.setUnread(unreadCount.getUnreadCount()); ? null
sub.setNewestItemTime(unreadCount.getNewestItemTime()); : feed.getDisabledUntil());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId())); sub.setUnread(unreadCount.getUnreadCount());
sub.setFilter(subscription.getFilter()); sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setFilterLegacy(subscription.getFilterLegacy()); sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setPushNotificationsEnabled(subscription.isPushNotificationsEnabled()); sub.setFilter(subscription.getFilter());
sub.setAutoMarkAsReadAfterDays(subscription.getAutoMarkAsReadAfterDays()); sub.setFilterLegacy(subscription.getFilterLegacy());
return sub; sub.setPushNotificationsEnabled(subscription.isPushNotificationsEnabled());
} sub.setAutoMarkAsReadAfterDays(subscription.getAutoMarkAsReadAfterDays());
return sub;
} }
}

Some files were not shown because too many files have changed in this diff Show More