Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-22 12:04:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

use google java formatter to avoid downloading eclipse during build

Browse Source
This commit is contained in:
Athou
2026-07-05 10:18:30 +02:00
parent f4dc279ec9
commit 1bd9b71215
200 changed files with 13923 additions and 12754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

401
commafeed-server/dev/EclipseCodeFormatter.xml
View File

@@ -1,401 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<profiles version="23">
<profile kind="CodeFormatterProfile" name="CommaFeed" version="23">
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.align_with_spaces" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
<setting id="org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_record_components" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_logical_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line" value="one_line_if_empty"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_shift_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_parameters" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_loops" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_switch_case_arrow_operator" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_enum_constant" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.text_block_indentation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_module_statements" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_permitted_types" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_annotations" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_assertion_message_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines" value="2147483647"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_not_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_type_arguments" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_permitted_types_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_record_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.javadoc_do_not_separate_block_tags" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_tag_description" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_record_constructor" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_string_concatenation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_record_components" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_additive_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_record_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.keep_switch_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_switch_case_with_arrow" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_method_body_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_additive_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_constructor" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_record_declaration_on_one_line" value="one_line_if_empty"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_lambda_body" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_relational_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.align_selector_in_method_invocation_on_expression_first_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_record_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_switch_case_with_arrow_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_switch_case_with_colon" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_conditional_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_shift_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.keep_code_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_record_components" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_assignment_operator" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_switch_case_with_arrow" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_record_constructor_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assertion_message" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_record_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_relational_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_logical_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_permitted_types" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="tab"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_string_concatenation" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
</profile>
</profiles>

23
commafeed-server/dev/checkstyle.xml
View File

@@ -1,8 +1,8 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker">
<property name="charset" value="UTF-8" />
<property name="fileExtensions" value="java" />
@@ -40,11 +40,6 @@
<module name="AvoidStarImport" />
<module name="AvoidStaticImport" />
<module name="IllegalImport" />
<module name="ImportOrder">
<property name="groups" value="/^java\./,javax,jakarta,org,com" />
<property name="ordered" value="true" />
<property name="separated" value="true" />
</module>
<module name="RedundantImport" />
<module name="UnusedImports" />
@@ -60,7 +55,7 @@
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="ignore|ignored" />
<message key="catch.block.empty"
value="Empty catch block. You can use the name 'ignore' or 'ignored' for the exception variable if you really want an empty catch block, but you should strongly consider at the very least logging something." />
value="Empty catch block. You can use the name 'ignore' or 'ignored' for the exception variable if you really want an empty catch block, but you should strongly consider at the very least logging something." />
</module>
<module name="LeftCurly" />
<module name="NeedBraces" />
@@ -76,7 +71,7 @@
<module name="FallThrough" />
<module name="IllegalInstantiation">
<property name="classes"
value="java.lang.Boolean, java.lang.Byte, java.lang.Character, java.lang.Double, java.lang.Float, java.lang.Integer, java.lang.Long, java.lang.Short" />
value="java.lang.Boolean, java.lang.Byte, java.lang.Character, java.lang.Double, java.lang.Float, java.lang.Integer, java.lang.Long, java.lang.Short" />
</module>
<module name="IllegalType" />
<module name="ModifiedControlVariable">
@@ -113,14 +108,6 @@
<module name="NoLineWrap" />
<module name="NoWhitespaceBefore" />
<module name="ParenPad" />
<module name="RegexpSinglelineJava">
<property name="format" value="^\t* +\t*\S" />
<property name="message" value="Line has leading space characters; indentation should be performed with tabs only." />
<property name="ignoreComments" value="true" />
</module>
<module name="WhitespaceAround">
<property name="allowEmptyTypes" value="true" />
</module>
</module>
</module>

7
commafeed-server/dev/eclipse.importorder
View File

@@ -1,7 +0,0 @@
#Organize Import Order
#Wed Jan 29 15:15:04 CET 2025
0=java
1=javax
2=jakarta
3=org
4=com

9
commafeed-server/pom.xml
View File

@@ -285,12 +285,9 @@
<configuration>
<encoding>UTF-8</encoding>
<java>
<eclipse>
<file>${project.basedir}/dev/EclipseCodeFormatter.xml</file>
</eclipse>
<importOrder>
<file>${project.basedir}/dev/eclipse.importorder</file>
</importOrder>
<googleJavaFormat>
<style>AOSP</style>
</googleJavaFormat>
</java>
</configuration>
</plugin>

42
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
View File

@@ -1,15 +1,13 @@
package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.ImageProxyUrl;
import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -18,28 +16,28 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class CommaFeedApplication {
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) {
log.info("starting up...");
public void start(@Observes StartupEvent ev) {
log.info("starting up...");
PasswordConstraintValidator.setMinimumPasswordLength(config.users().minimumPasswordLength());
PasswordConstraintValidator.setMinimumPasswordLength(
config.users().minimumPasswordLength());
if (config.imageProxyEnabled()) {
ImageProxyUrl.generateKey();
}
if (config.imageProxyEnabled()) {
ImageProxyUrl.generateKey();
}
feedRefreshEngine.start();
taskScheduler.start();
}
feedRefreshEngine.start();
taskScheduler.start();
}
public void stop(@Observes ShutdownEvent ev) {
log.info("shutting down...");
feedRefreshEngine.stop();
taskScheduler.stop();
}
public void stop(@Observes ShutdownEvent ev) {
log.info("shutting down...");
feedRefreshEngine.stop();
taskScheduler.stop();
}
}

622
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -1,406 +1,360 @@
package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
/**
* CommaFeed configuration
*
* Default values are for production, they can be overridden in application.properties for other profiles
* <p>Default values are for production, they can be overridden in application.properties for other
* profiles
*/
@ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration {
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser.
*
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed
* directly by the browser.
*
* <p>This is useful if commafeed is accessed through a restricting proxy that blocks some feeds
* that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* Enable password recovery via email.
*
* Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Enable password recovery via email.
*
* <p>Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Message displayed in a notification at the bottom of the page.
*/
Optional<String> announcement();
/** Message displayed in a notification at the bottom of the page. */
Optional<String> announcement();
/**
* Google Auth key for fetching Youtube channel favicons.
*/
Optional<String> googleAuthKey();
/** Google Auth key for fetching Youtube channel favicons. */
Optional<String> googleAuthKey();
/**
* HTTP client configuration
*/
@ConfigDocSection
HttpClient httpClient();
/** HTTP client configuration */
@ConfigDocSection
HttpClient httpClient();
/**
* Feed refresh engine settings.
*/
@ConfigDocSection
FeedRefresh feedRefresh();
/** Feed refresh engine settings. */
@ConfigDocSection
FeedRefresh feedRefresh();
/**
* Push notification settings.
*/
@ConfigDocSection
PushNotifications pushNotifications();
/** Push notification settings. */
@ConfigDocSection
PushNotifications pushNotifications();
/**
* Database settings.
*/
@ConfigDocSection
Database database();
/** Database settings. */
@ConfigDocSection
Database database();
/**
* Users settings.
*/
@ConfigDocSection
Users users();
/** Users settings. */
@ConfigDocSection
Users users();
/**
* Websocket settings.
*/
@ConfigDocSection
Websocket websocket();
/** Websocket settings. */
@ConfigDocSection
Websocket websocket();
/**
* Duration to wait for the feed refresh engine and the task scheduler to stop when the application is shutting down.
*/
@WithDefault("2s")
Duration shutdownTimeout();
/**
* Duration to wait for the feed refresh engine and the task scheduler to stop when the
* application is shutting down.
*/
@WithDefault("2s")
Duration shutdownTimeout();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
/**
* Time to wait for a connection to be established.
*/
@WithDefault("5s")
Duration connectTimeout();
/** Time to wait for a connection to be established. */
@WithDefault("5s")
Duration connectTimeout();
/**
* Time to wait for SSL handshake to complete.
*/
@WithDefault("5s")
Duration sslHandshakeTimeout();
/** Time to wait for SSL handshake to complete. */
@WithDefault("5s")
Duration sslHandshakeTimeout();
/**
* Time to wait between two packets before timeout.
*/
@WithDefault("10s")
Duration socketTimeout();
/** Time to wait between two packets before timeout. */
@WithDefault("10s")
Duration socketTimeout();
/**
* Time to wait for the full response to be received.
*/
@WithDefault("10s")
Duration responseTimeout();
/** Time to wait for the full response to be received. */
@WithDefault("10s")
Duration responseTimeout();
/**
* Time to live for a connection in the pool.
*/
@WithDefault("30s")
Duration connectionTimeToLive();
/** Time to live for a connection in the pool. */
@WithDefault("30s")
Duration connectionTimeToLive();
/**
* Time between eviction runs for idle connections.
*/
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/** Time between eviction runs for idle connections. */
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while
* parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal
* resources.
*
* You may want to enable this if you host a public instance of CommaFeed with registrations open.
*/
@WithDefault("false")
boolean blockLocalAddresses();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks,
* which could potentially expose internal resources.
*
* <p>You may want to enable this if you host a public instance of CommaFeed with
* registrations open.
*/
@WithDefault("false")
boolean blockLocalAddresses();
/**
* HTTP client cache configuration
*/
@ConfigDocSection
HttpClientCache cache();
}
/** HTTP client cache configuration */
@ConfigDocSection
HttpClientCache cache();
}
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the
* first time or when clicking "fetch all my feeds now").
*/
@WithDefault("true")
boolean enabled();
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts
* (e.g. when subscribing to a feed for the first time or when clicking "fetch all my feeds
* now").
*/
@WithDefault("true")
boolean enabled();
/**
* Maximum amount of memory the cache can use.
*/
@WithDefault("10M")
MemorySize maximumMemorySize();
/** Maximum amount of memory the cache can use. */
@WithDefault("10M")
MemorySize maximumMemorySize();
/**
* Duration after which an entry is removed from the cache.
*/
@WithDefault("1m")
Duration expiration();
}
/** Duration after which an entry is removed from the cache. */
@WithDefault("1m")
Duration expiration();
}
interface FeedRefresh {
/**
* Default amount of time CommaFeed will wait before refreshing a feed.
*/
@WithDefault("5m")
Duration interval();
interface FeedRefresh {
/** Default amount of time CommaFeed will wait before refreshing a feed. */
@WithDefault("5m")
Duration interval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li>
* <li>we receive a Cache-Control header from the feed</li>
* <li>we receive a Retry-After header from the feed</li>
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an
* upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially
* <li>we receive a Cache-Control header from the feed
* <li>we receive a Retry-After header from the feed
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since
* the last entry was published. The interval will be sometimes between the default refresh interval
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average
* time between entries and the time since the last entry was published. The interval will
* be sometimes between the default refresh interval (`commafeed.feed-refresh.interval`) and
* the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* <p>See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* Feed refresh engine error handling settings.
*/
@ConfigDocSection
FeedRefreshErrorHandling errors();
/** Feed refresh engine error handling settings. */
@ConfigDocSection
FeedRefreshErrorHandling errors();
/**
* Amount of http threads used to fetch feeds.
*/
@Min(1)
@WithDefault("3")
int httpThreads();
/** Amount of http threads used to fetch feeds. */
@Min(1)
@WithDefault("3")
int httpThreads();
/**
* Amount of threads used to insert new entries in the database.
*/
@Min(1)
@WithDefault("1")
int databaseThreads();
/** Amount of threads used to insert new entries in the database. */
@Min(1)
@WithDefault("1")
int databaseThreads();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again.
*
* 0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not
* refreshed until they log in again.
*
* <p>0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is
* considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to
* avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
interface PushNotifications {
/**
* Whether to enable push notifications to notify users of new entries in their feeds.
*/
@WithDefault("true")
boolean enabled();
interface PushNotifications {
/** Whether to enable push notifications to notify users of new entries in their feeds. */
@WithDefault("true")
boolean enabled();
/**
* Amount of threads used to send external notifications about new entries.
*/
@Min(1)
@WithDefault("5")
int threads();
/** Amount of threads used to send external notifications about new entries. */
@Min(1)
@WithDefault("5")
int threads();
/**
* Maximum amount of notifications that can be queued before new notifications are discarded.
*/
@Min(1)
@WithDefault("100")
int queueCapacity();
}
/**
* Maximum amount of notifications that can be queued before new notifications are
* discarded.
*/
@Min(1)
@WithDefault("100")
int queueCapacity();
}
interface FeedRefreshErrorHandling {
/**
* Number of retries before backoff is applied.
*/
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
interface FeedRefreshErrorHandling {
/** Number of retries before backoff is applied. */
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of
* errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
interface Database {
/**
* Timeout applied to all database queries.
*
* 0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
interface Database {
/**
* Timeout applied to all database queries.
*
* <p>0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
/**
* Database cleanup settings.
*/
@ConfigDocSection
Cleanup cleanup();
/** Database cleanup settings. */
@ConfigDocSection
Cleanup cleanup();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* 0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* <p>0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted.
*
* 0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will
* be deleted.
*
* <p>0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum number of entries per feed to keep in the database.
*
* 0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Maximum number of entries per feed to keep in the database.
*
* <p>0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Limit the number of feeds a user can subscribe to.
*
* 0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Limit the number of feeds a user can subscribe to.
*
* <p>0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Rows to delete per query while cleaning up old entries.
*/
@Positive
@WithDefault("100")
int batchSize();
/** Rows to delete per query while cleaning up old entries. */
@Positive
@WithDefault("100")
int batchSize();
/**
* Whether to keep starred entries when cleaning up old entries.
*/
@WithDefault("true")
boolean keepStarredEntries();
/** Whether to keep starred entries when cleaning up old entries. */
@WithDefault("true")
boolean keepStarredEntries();
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null;
}
}
}
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0
? Instant.now().minus(statusesMaxAge())
: null;
}
}
}
interface Users {
/**
* Whether to let users create accounts for themselves.
*/
@WithDefault("false")
boolean allowRegistrations();
interface Users {
/** Whether to let users create accounts for themselves. */
@WithDefault("false")
boolean allowRegistrations();
/**
* Minimum password length for user accounts.
*/
@WithDefault("4")
int minimumPasswordLength();
/** Minimum password length for user accounts. */
@WithDefault("4")
int minimumPasswordLength();
/**
* Whether an email address is required when creating a user account.
*/
@WithDefault("false")
boolean emailAddressRequired();
/** Whether an email address is required when creating a user account. */
@WithDefault("false")
boolean emailAddressRequired();
/**
* Whether to create a demo account the first time the app starts.
*/
@WithDefault("false")
boolean createDemoAccount();
}
/** Whether to create a demo account the first time the app starts. */
@WithDefault("false")
boolean createDemoAccount();
}
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds.
*/
@WithDefault("true")
boolean enabled();
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new
* entries for feeds.
*/
@WithDefault("true")
boolean enabled();
/**
* Interval at which the client will send a ping message on the websocket to keep the connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
/**
* Interval at which the client will send a ping message on the websocket to keep the
* connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload
* the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
}

2
commafeed-server/src/main/java/com/commafeed/CommaFeedConstants.java
View File

@@ -4,5 +4,5 @@ import lombok.experimental.UtilityClass;
@UtilityClass
public class CommaFeedConstants {
public static final String USERNAME_DEMO = "demo";
public static final String USERNAME_DEMO = "demo";
}

27
commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
View File

@@ -1,25 +1,22 @@
package com.commafeed;
import java.time.InstantSource;
import com.codahale.metrics.MetricRegistry;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import com.codahale.metrics.MetricRegistry;
import java.time.InstantSource;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}

29
commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
View File

@@ -1,30 +1,27 @@
package com.commafeed;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter;
@Singleton
@Getter
public class CommaFeedVersion {
private final String version;
private final String gitCommit;
private final String version;
private final String gitCommit;
public CommaFeedVersion() throws IOException {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
public CommaFeedVersion() throws IOException {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
}

60
commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
View File

@@ -1,52 +1,54 @@
package com.commafeed;
import com.commafeed.security.CookieService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import jakarta.annotation.Priority;
import jakarta.validation.ValidationException;
import jakarta.ws.rs.core.NewCookie;
import jakarta.ws.rs.ext.Provider;
import lombok.RequiredArgsConstructor;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import com.commafeed.security.CookieService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Provider
@Priority(1)
public class ExceptionMappers {
private final CookieService cookieService;
private final CommaFeedConfiguration config;
private final CookieService cookieService;
private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(Status.UNAUTHORIZED, new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(
Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) {
NewCookie logoutCookie = cookieService.buildLogoutCookie();
return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage())).cookie(logoutCookie).build();
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(
AuthenticationFailedException e) {
NewCookie logoutCookie = cookieService.buildLogoutCookie();
return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()))
.cookie(logoutCookie)
.build();
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {}
@RegisterForReflection
public record AuthenticationFailed(String message) {}
@RegisterForReflection
public record AuthenticationFailed(String message) {}
@RegisterForReflection
public record ValidationFailed(String message) {}
@RegisterForReflection
public record ValidationFailed(String message) {}
}

28
commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
View File

@@ -1,29 +1,27 @@
package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
@Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// read and write instants as milliseconds instead of nanoseconds
objectMapper
.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
}

544
commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
View File

@@ -6,221 +6,343 @@ import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection(
targets = {
// metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class,
targets = {
// metrics
MetricRegistry.class,
Meter.class,
Gauge.class,
Counter.class,
Timer.class,
Histogram.class,
// rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome
java.util.Date.class,
com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class,
com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class,
com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class,
com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class,
com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class,
com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class,
com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class,
com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class,
com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class,
com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class,
com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class,
com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class,
com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class,
com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class,
com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class,
com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class,
com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class,
com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class,
com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class,
com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class,
com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class,
com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class,
com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class,
com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class,
com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class,
com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class,
com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class,
com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class,
com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class,
com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class,
com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class,
com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class,
com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class,
com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class,
com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class,
com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class,
com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class,
com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class,
com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class,
com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class,
com.rometools.rome.feed.rss.Image.class,
com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class,
com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class,
com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class,
com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class,
com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class,
com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class,
com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class,
com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class,
com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class,
com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class,
com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class,
com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class,
com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class,
com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class,
com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class,
com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class,
com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class,
com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class,
com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, })
public class NativeImageClasses {
}
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class,
com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class,
com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class,
com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class,
com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class,
com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class,
com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class,
com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class,
com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class,
com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class,
com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class,
com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class,
com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class,
com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class,
com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class,
com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML20.class,
})
public class NativeImageClasses {}

28
commafeed-server/src/main/java/com/commafeed/backend/Digests.java
View File

@@ -1,29 +1,27 @@
package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import java.nio.charset.StandardCharsets;
import lombok.experimental.UtilityClass;
@UtilityClass
@SuppressWarnings("deprecation")
public class Digests {
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
}

246
commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java
View File

@@ -1,5 +1,12 @@
package com.commafeed.backend;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.net.HttpHeaders;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressNetwork;
import inet.ipaddr.IPAddressString;
import jakarta.inject.Singleton;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
@@ -8,9 +15,9 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.SequencedMap;
import java.util.zip.GZIPInputStream;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig;
@@ -35,137 +42,148 @@ import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.brotli.dec.BrotliInputStream;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.net.HttpHeaders;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressNetwork;
import inet.ipaddr.IPAddressString;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
@Singleton
@RequiredArgsConstructor
public class HttpClientFactory {
private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
private final CommaFeedConfiguration config;
private final CommaFeedVersion version;
private final CommaFeedConfiguration config;
private final CommaFeedVersion version;
public CloseableHttpClient newClient(int poolSize) {
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config, poolSize);
String userAgent = config.httpClient()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
return newClient(config, connectionManager, userAgent);
}
public CloseableHttpClient newClient(int poolSize) {
PoolingHttpClientConnectionManager connectionManager =
newConnectionManager(config, poolSize);
String userAgent =
config.httpClient()
.userAgent()
.orElseGet(
() ->
String.format(
"CommaFeed/%s (https://github.com/Athou/commafeed)",
version.getVersion()));
return newClient(config, connectionManager, userAgent);
}
private CloseableHttpClient newClient(CommaFeedConfiguration config, HttpClientConnectionManager connectionManager, String userAgent) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
private CloseableHttpClient newClient(
CommaFeedConfiguration config,
HttpClientConnectionManager connectionManager,
String userAgent) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>();
contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>();
contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
RedirectStrategy redirectStrategy = config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
: new DefaultRedirectStrategy();
RedirectStrategy redirectStrategy =
config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
: new DefaultRedirectStrategy();
return HttpClientBuilder.create()
.disableConnectionState()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
.setRedirectStrategy(redirectStrategy)
.build();
}
return HttpClientBuilder.create()
.disableConnectionState()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(
TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
.setRedirectStrategy(redirectStrategy)
.build();
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config, int poolSize) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
DnsResolver dnsResolver = config.httpClient().blockLocalAddresses() ? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
: DNS_RESOLVER;
private PoolingHttpClientConnectionManager newConnectionManager(
CommaFeedConfiguration config, int poolSize) {
SSLFactory sslFactory =
SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
DnsResolver dnsResolver =
config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
: DNS_RESOLVER;
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(
ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(
Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(
TlsConfig.custom()
.setHandshakeTimeout(
Timeout.of(config.httpClient().sslHandshakeTimeout()))
.build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
}
private static boolean isLocalAddress(InetAddress address) {
IPAddress ip = new IPAddressNetwork.IPAddressGenerator().from(address);
return ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip);
}
private static boolean isLocalAddress(InetAddress address) {
IPAddress ip = new IPAddressNetwork.IPAddressGenerator().from(address);
return ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip);
}
private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver {
@Override
public InetAddress[] resolve(String host) throws UnknownHostException {
InetAddress[] addresses = delegate.resolve(host);
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new UnknownHostException(
"Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
}
private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver {
@Override
public InetAddress[] resolve(String host) throws UnknownHostException {
InetAddress[] addresses = delegate.resolve(host);
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new UnknownHostException("Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
}
@Override
public String resolveCanonicalHostname(String host) throws UnknownHostException {
return delegate.resolveCanonicalHostname(host);
}
}
@Override
public String resolveCanonicalHostname(String host) throws UnknownHostException {
return delegate.resolveCanonicalHostname(host);
}
}
@RequiredArgsConstructor
private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
@RequiredArgsConstructor
private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
private final DnsResolver delegate;
private final DnsResolver delegate;
@Override
public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context)
throws HttpException {
URI redirectUri = super.getLocationURI(request, response, context);
@Override
public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException {
URI redirectUri = super.getLocationURI(request, response, context);
String host = redirectUri.getHost();
if (host == null) {
throw new HttpException("Redirect URI does not have a host: " + redirectUri);
}
String host = redirectUri.getHost();
if (host == null) {
throw new HttpException("Redirect URI does not have a host: " + redirectUri);
}
InetAddress[] addresses;
try {
addresses = delegate.resolve(host);
} catch (UnknownHostException e) {
throw new HttpException("Unknown host: " + host);
}
InetAddress[] addresses;
try {
addresses = delegate.resolve(host);
} catch (UnknownHostException e) {
throw new HttpException("Unknown host: " + host);
}
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new HttpException("Access to local address blocked: " + addr.getHostAddress());
}
}
return redirectUri;
}
}
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new HttpException(
"Access to local address blocked: " + addr.getHostAddress());
}
}
return redirectUri;
}
}
}

524
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -1,5 +1,15 @@
package com.commafeed.backend;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
@@ -8,10 +18,12 @@ import java.time.Instant;
import java.time.InstantSource;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.config.RequestConfig;
@@ -27,282 +39,326 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
/** Smart HTTP getter: handles gzip, ssl, last modified and etag headers */
@Singleton
@Slf4j
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, HttpClientFactory httpClientFactory,
MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
this.cache = newCache(config);
public HttpGetter(
CommaFeedConfiguration config,
InstantSource instantSource,
HttpClientFactory httpClientFactory,
MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> ArrayUtils.getLength(e.content)).sum());
}
metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "size"),
() -> cache == null ? 0 : cache.size());
metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() ->
cache == null
? 0
: cache.asMap().values().stream()
.mapToInt(e -> ArrayUtils.getLength(e.content))
.sum());
}
public HttpResult get(String url) throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(String url)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
public HttpResult get(HttpRequest request)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw Lombok.sneakyThrow(e);
}
}
}
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw Lombok.sneakyThrow(e);
}
}
}
int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
throw new TooManyRequestsException(response.retryAfter());
}
int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS
|| code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
throw new TooManyRequestsException(response.retryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.eTagHeader();
String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.eTagHeader();
Duration validFor = Optional.ofNullable(response.cacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
Duration validFor =
Optional.ofNullable(response.cacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.content(), response.contentType(), lastModifiedHeader, eTagHeader, response.urlAfterRedirect(),
validFor);
}
return new HttpResult(
response.content(),
response.contentType(),
lastModifiedHeader,
eTagHeader,
response.urlAfterRedirect(),
validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> {
byte[] content = resp.getEntity() == null ? null
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
return client.execute(
request.toClassicHttpRequest(),
context,
resp -> {
byte[] content =
resp.getEntity() == null
? null
: toByteArray(
resp.getEntity(),
config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
CacheControl cacheControl =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
Instant retryAfter =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
String contentType =
Optional.ofNullable(resp.getEntity())
.map(HttpEntity::getContentType)
.orElse(null);
String urlAfterRedirect =
Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
return new HttpResponse(
code,
lastModifiedHeader,
eTagHeader,
cacheControl,
retryAfter,
content,
contentType,
urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)"
.formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) {
throw new IOException(
"Response size exceeds the maximum allowed size (%s bytes)"
.formatted(maxBytes));
}
return bytes;
}
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
return CacheBuilder.newBuilder()
.weigher(
(HttpRequest key, HttpResponse value) ->
value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
/**
* if the value of this header changed, this is its new value
*/
private final String newLastModifiedHeader;
/** if the value of this header changed, this is its new value */
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private final String newEtagHeader;
/** if the value of this header changed, this is its new value */
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
public NotModifiedException(
String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
private final int code;
private final int code;
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
private record HttpResponse(int code, String lastModifiedHeader, String eTagHeader, CacheControl cacheControl, Instant retryAfter,
byte[] content, String contentType, String urlAfterRedirect) {}
public record HttpResult(byte[] content, String contentType, String lastModifiedSince, String eTag, String urlAfterRedirect,
Duration validFor) {}
private record HttpResponse(
int code,
String lastModifiedHeader,
String eTagHeader,
CacheControl cacheControl,
Instant retryAfter,
byte[] content,
String contentType,
String urlAfterRedirect) {}
public record HttpResult(
byte[] content,
String contentType,
String lastModifiedSince,
String eTag,
String urlAfterRedirect,
Duration validFor) {}
}

155
commafeed-server/src/main/java/com/commafeed/backend/Urls.java
View File

@@ -2,105 +2,104 @@ package com.commafeed.backend;
import java.net.URI;
import java.util.regex.Pattern;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Strings;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@UtilityClass
@Slf4j
public class Urls {
private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?"));
private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?"));
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsolute(String url) {
return isHttp(url) || isHttps(url);
}
public static boolean isAbsolute(String url) {
return isHttp(url) || isHttps(url);
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
/**
* @param relativeUrl the url of the entry
* @param feedLink the url of the feed as described in the feed
* @param feedUrl the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return URI.create(baseUrl).resolve(relativeUrl).toString();
} catch (IllegalArgumentException e) {
log.debug("Unable to create absolute url from relative url: {} base: {}", relativeUrl, baseUrl, e);
return null;
}
}
try {
return URI.create(baseUrl).resolve(relativeUrl).toString();
} catch (IllegalArgumentException e) {
log.debug(
"Unable to create absolute url from relative url: {} base: {}",
relativeUrl,
baseUrl,
e);
return null;
}
}
public static String removeTrailingSlash(String url) {
if (url == null) {
return null;
}
public static String removeTrailingSlash(String url) {
if (url == null) {
return null;
}
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalize(String url) {
if (url == null) {
return null;
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to
* identify a feed and avoid duplicates
*/
public static String normalize(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = QUESTION_MARK.split(normalized)[0];
normalized = Strings.CS.removeEnd(normalized, "/");
}
// feedburner feeds have a special treatment
if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = QUESTION_MARK.split(normalized)[0];
normalized = Strings.CS.removeEnd(normalized, "/");
}
return normalized;
}
return normalized;
}
}

109
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
View File

@@ -1,71 +1,76 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Objects;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch();
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user))
.join(CATEGORY.user, QUser.user)
.fetchJoin()
.fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user), CATEGORY.id.eq(id))
.fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate)
.fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}

93
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -1,64 +1,69 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.Strings;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.Strings;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query =
query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(
FEED.disabledUntil
.isNull()
.or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query()
.selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch();
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED)
.where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists())
.limit(max)
.fetch();
}
}

48
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
View File

@@ -1,36 +1,38 @@
package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch();
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT)
.from(CONTENT)
.where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash))
.fetch();
}
public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
public long deleteWithoutEntries(int max) {
List<Long> ids =
query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
}

144
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,14 +1,5 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
@@ -16,83 +7,98 @@ import com.google.common.collect.Lists;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY)
.from(ENTRY)
.where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed))
.limit(1)
.fetchOne();
}
public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
if (guidHashes.isEmpty()) {
return Set.of();
}
public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
if (guidHashes.isEmpty()) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch)).fetch());
}
return result;
}
Set<String> result = new HashSet<>();
for (List<String> batch :
Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(
query().select(ENTRY.guidHash)
.from(ENTRY)
.where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch))
.fetch());
}
return result;
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);
public List<FeedCapacity> findFeedsExceedingCapacity(
long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
return query.groupBy(ENTRY.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch()
.stream()
.map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
.toList();
}
return query.groupBy(ENTRY.feed).having(count.gt(maxCapacity)).limit(max).fetch().stream()
.map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
.toList();
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
public int delete(Long feedId, long max) {
List<FeedEntry> list =
query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query = query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max);
/** Delete entries older than a certain date */
public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query =
query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
return delete(query.fetch());
}
return delete(query.fetch());
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max);
/** Delete the oldest entries of a feed */
public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query =
query().selectFrom(ENTRY)
.where(ENTRY.feed.id.eq(feedId))
.orderBy(ENTRY.published.asc())
.limit(max);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
return delete(query.fetch());
}
return delete(query.fetch());
}
public record FeedCapacity(Long id, Long capacity) {}
public record FeedCapacity(Long id, Long capacity) {}
}

492
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
@@ -32,273 +21,316 @@ import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeed FEED = QFeed.feed;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeed FEED = QFeed.feed;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatusDAO(
EntityManager entityManager,
FeedEntryTagDAO feedEntryTagDAO,
CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
return handleStatus(user, status, sub, entry);
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses =
query().selectFrom(STATUS)
.where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub))
.fetch();
FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
return handleStatus(user, status, sub, entry);
}
/**
* creates an artificial "unread" status if status is null
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
/** creates an artificial "unread" status if status is null */
private FeedEntryStatus handleStatus(
User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold =
config.database().cleanup().statusesInstantThreshold();
boolean read =
statusesInstantThreshold != null
&& entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user,
statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds =
feedEntryTagDAO.findByEntries(
user, statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
public List<FeedEntryStatus> findStarred(User user, List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit,
ReadingOrder order, boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content, CONTENT).fetchJoin();
}
public List<FeedEntryStatus> findStarred(
User user,
List<FeedEntryKeyword> keywords,
Instant newerThan,
int offset,
int limit,
ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query =
query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.ASC) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (order == ReadingOrder.ASC) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (offset > -1) {
query.offset(offset);
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
return statuses;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
String tag, Long minEntryId, Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
public List<FeedEntryStatus> findBySubscriptions(
User user,
List<FeedSubscription> subs,
boolean unreadOnly,
List<FeedEntryKeyword> keywords,
Instant newerThan,
int offset,
int limit,
ReadingOrder order,
boolean includeContent,
String tag,
Long minEntryId,
Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId =
subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (order != null) {
if (order == ReadingOrder.ASC) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (order != null) {
if (order == ReadingOrder.ASC) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
if (includeContent) {
fetchTags(user, statuses);
}
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
return statuses;
}
private void applyKeywordsFilter(JPAQuery<?> query, List<FeedEntryKeyword> keywords) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.keyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.keyword()));
if (keyword.mode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
private void applyKeywordsFilter(JPAQuery<?> query, List<FeedEntryKeyword> keywords) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.keyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.keyword()));
if (keyword.mode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query =
query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids =
query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
public long autoMarkAsRead(int limit) {
Instant now = Instant.now();
public long autoMarkAsRead(int limit) {
Instant now = Instant.now();
BooleanBuilder where = new BooleanBuilder();
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull());
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
BooleanBuilder where = new BooleanBuilder();
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull());
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
NumberExpression<Integer> daysDiff = Expressions.numberTemplate(Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published,
now);
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
NumberExpression<Integer> daysDiff =
Expressions.numberTemplate(
Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, now);
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
where.and(buildUnreadPredicate());
where.and(buildUnreadPredicate());
List<Tuple> tuples = query().select(ENTRY, STATUS, SUBSCRIPTION)
.from(ENTRY)
.join(ENTRY.feed, FEED)
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(SUBSCRIPTION))
.where(where)
.limit(limit)
.fetch();
List<Tuple> tuples =
query().select(ENTRY, STATUS, SUBSCRIPTION)
.from(ENTRY)
.join(ENTRY.feed, FEED)
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(SUBSCRIPTION))
.where(where)
.limit(limit)
.fetch();
long updated = 0;
long updated = 0;
// Update existing statuses
List<Long> statusIdsToUpdate = tuples.stream()
.map(t -> t.get(STATUS))
.filter(s -> s != null && s.getId() != null)
.map(FeedEntryStatus::getId)
.distinct()
.toList();
// Update existing statuses
List<Long> statusIdsToUpdate =
tuples.stream()
.map(t -> t.get(STATUS))
.filter(s -> s != null && s.getId() != null)
.map(FeedEntryStatus::getId)
.distinct()
.toList();
if (!statusIdsToUpdate.isEmpty()) {
updated += updateQuery(STATUS).where(STATUS.id.in(statusIdsToUpdate)).set(STATUS.read, true).execute();
}
if (!statusIdsToUpdate.isEmpty()) {
updated +=
updateQuery(STATUS)
.where(STATUS.id.in(statusIdsToUpdate))
.set(STATUS.read, true)
.execute();
}
// Insert new statuses for entries without existing status
for (Tuple tuple : tuples) {
FeedEntryStatus status = tuple.get(STATUS);
if (status == null || status.getId() == null) {
FeedEntry entry = tuple.get(ENTRY);
FeedSubscription sub = tuple.get(SUBSCRIPTION);
FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry);
newStatus.setRead(true);
persist(newStatus);
updated++;
}
}
return updated;
}
// Insert new statuses for entries without existing status
for (Tuple tuple : tuples) {
FeedEntryStatus status = tuple.get(STATUS);
if (status == null || status.getId() == null) {
FeedEntry entry = tuple.get(ENTRY);
FeedSubscription sub = tuple.get(SUBSCRIPTION);
FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry);
newStatus.setRead(true);
persist(newStatus);
updated++;
}
}
return updated;
}
}

47
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
View File

@@ -1,39 +1,38 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query()
.selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}

216
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
View File

@@ -1,20 +1,5 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
@@ -23,109 +8,136 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.querydsl.jpa.JPQLQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager;
private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager
.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(
new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count())
.from(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query =
query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList();
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds =
categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream()
.filter(
s ->
s.getCategory() != null
&& categoryIds.contains(s.getCategory().getId()))
.toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}

81
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
View File

@@ -1,66 +1,61 @@
package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import jakarta.persistence.EntityManager;
import java.time.Duration;
import java.util.Collection;
import lombok.RequiredArgsConstructor;
import org.hibernate.jpa.SpecHints;
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager;
private final Class<T> entityClass;
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
public void persist(T model) {
entityManager.persist(model);
}
public void persist(T model) {
entityManager.persist(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
}

12
commafeed-server/src/main/java/com/commafeed/backend/dao/Predicates.java
View File

@@ -4,15 +4,17 @@ import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.querydsl.core.types.dsl.BooleanExpression;
import com.querydsl.jpa.JPAExpressions;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Predicates {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
public static BooleanExpression isNotStarred(QFeedEntry entry) {
return JPAExpressions.selectOne().from(STATUS).where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue())).notExists();
}
public static BooleanExpression isNotStarred(QFeedEntry entry) {
return JPAExpressions.selectOne()
.from(STATUS)
.where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue()))
.notExists();
}
}

18
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
View File

@@ -1,19 +1,17 @@
package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction;
import jakarta.inject.Singleton;
import java.util.concurrent.Callable;
@Singleton
public class UnitOfWork {
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
}

37
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
View File

@@ -1,33 +1,32 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
View File

@@ -1,39 +1,37 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
public long countAdmins() {
return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne();
}
public long countAdmins() {
return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne();
}
}

19
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
View File

@@ -1,22 +1,21 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
}

89
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -1,21 +1,17 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
@Slf4j
@RequiredArgsConstructor
@@ -23,44 +19,49 @@ import lombok.extern.slf4j.Slf4j;
@Priority(3)
public class FacebookFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
String iconUrl =
String.format(
"https://graph.facebook.com/%s/picture?type=square&height=16", userName);
try {
log.debug("Getting Facebook user's icon, {}", url);
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
return null;
}
}
HttpResult iconResult = getter.get(iconUrl);
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
return null;
}
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream()
.filter(p -> "id".equals(p.getName()))
.map(NameValuePair::getValue)
.findFirst()
.orElse(null);
}
}

28
commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
View File

@@ -1,24 +1,24 @@
package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public record Favicon(byte[] icon, MediaType mediaType) {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType, e);
return DEFAULT_MEDIA_TYPE;
}
}
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug(
"invalid content type '{}' received, returning default value", contentType, e);
return DEFAULT_MEDIA_TYPE;
}
}
}

3
commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java
View File

@@ -4,6 +4,5 @@ import com.commafeed.backend.model.Feed;
public interface FaviconFetcher {
Favicon fetch(Feed feed);
Favicon fetch(Feed feed);
}

40
commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java
View File

@@ -1,39 +1,35 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetch favicon from the url declared in the feed.
*/
/** Fetch favicon from the url declared in the feed. */
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(2)
public class FeedFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
}

83
commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java
View File

@@ -1,65 +1,60 @@
package com.commafeed.backend.favicon;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Extracts favicon url from html page.
*/
/** Extracts favicon url from html page. */
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(1)
public class HtmlFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java
View File

@@ -1,45 +1,40 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import java.net.URI;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
*/
/** Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico) */
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(0)
public class RootFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
try {
URI uri = URI.create(url.trim());
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
try {
URI uri = URI.create(url.trim());
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
}

210
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -1,18 +1,5 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
@@ -23,9 +10,18 @@ import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
@Slf4j
@RequiredArgsConstructor
@@ -33,99 +29,121 @@ import lombok.extern.slf4j.Slf4j;
@Priority(3)
public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final String PART_SNIPPET = "snippet";
private static final String PART_SNIPPET = "snippet";
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private static final JsonPointer CHANNEL_THUMBNAIL_URL =
JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID =
JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("user"))
.findFirst();
Optional<NameValuePair> channelId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id"))
.findFirst();
Optional<NameValuePair> playlistId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id"))
.findFirst();
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
return null;
}
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
return null;
}
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).content();
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).content();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return new byte[0];
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return new byte[0];
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
View File

@@ -2,30 +2,28 @@ package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
/**
* A keyword used in a search query
*/
/** A keyword used in a search query */
public record FeedEntryKeyword(String keyword, Mode mode) {
public enum Mode {
INCLUDE, EXCLUDE
}
public enum Mode {
INCLUDE,
EXCLUDE
}
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

196
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpRequest;
@@ -22,96 +11,139 @@ import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
/**
* Fetches a feed then parses it
*/
/** Fetches a feed then parses it */
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcher(
FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
public FeedFetcherResult fetch(
String feedUrl,
boolean extractFeedUrlFromHtml,
String lastModified,
String eTag,
Instant lastPublishedDate,
String lastContentHash)
throws FeedParsingException,
IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException,
NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.content();
HttpResult result =
getter.get(
HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.content();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl =
extractFeedUrl(
urlProviders,
feedUrl,
new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.content();
parserResult = parser.parse(result.urlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
result =
getter.get(
HttpRequest.builder(extractedUrl)
.lastModified(lastModified)
.eTag(eTag)
.build());
content = result.content();
parserResult = parser.parse(result.urlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
boolean lastModifiedHeaderValueChanged =
!Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException(
"content hash not modified",
lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null
&& lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException(
"publishedDate not modified",
lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
return new FeedFetcherResult(parserResult, result.urlAfterRedirect(), result.lastModifiedSince(), result.eTag(), hash,
result.validFor());
}
return new FeedFetcherResult(
parserResult,
result.urlAfterRedirect(),
result.lastModifiedSince(),
result.eTag(),
hash,
result.validFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
return urlProviders.stream()
.flatMap(provider -> provider.get(url, urlContent).stream())
.filter(StringUtils::isNotBlank)
.findFirst()
.orElse(null);
}
private static String extractFeedUrl(
List<FeedURLProvider> urlProviders, String url, String urlContent) {
return urlProviders.stream()
.flatMap(provider -> provider.get(url, urlContent).stream())
.filter(StringUtils::isNotBlank)
.findFirst()
.orElse(null);
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {}
public record FeedFetcherResult(
FeedParserResult feed,
String urlAfterRedirect,
String lastModifiedHeader,
String lastETagHeader,
String contentHash,
Duration validFor) {}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}

468
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,19 +1,5 @@
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
@@ -25,229 +11,297 @@ import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final FeedUpdateNotifier notifier;
private final CommaFeedConfiguration config;
private final Meter refill;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final FeedUpdateNotifier notifier;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final BlockingDeque<Feed> queue;
private ExecutorService feedProcessingLoopExecutor;
private ExecutorService refillLoopExecutor;
private ThreadPoolExecutor refillExecutor;
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor databaseUpdaterExecutor;
private ThreadPoolExecutor notifierExecutor;
private ExecutorService feedProcessingLoopExecutor;
private ExecutorService refillLoopExecutor;
private ThreadPoolExecutor refillExecutor;
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor databaseUpdaterExecutor;
private ThreadPoolExecutor notifierExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
FeedUpdateNotifier notifier, CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.notifier = notifier;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
public FeedRefreshEngine(
UnitOfWork unitOfWork,
FeedDAO feedDAO,
FeedRefreshWorker worker,
FeedRefreshUpdater updater,
FeedUpdateNotifier notifier,
CommaFeedConfiguration config,
MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.notifier = notifier;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.queue = new LinkedBlockingDeque<>();
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) () -> workerExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "updater", "active"),
(Gauge<Integer>) () -> databaseUpdaterExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "notifier", "active"), (Gauge<Integer>) () -> notifierExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "notifier", "queue"), (Gauge<Integer>) () -> notifierExecutor.getQueue().size());
}
metrics.register(
MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(
MetricRegistry.name(getClass(), "worker", "active"),
(Gauge<Integer>) () -> workerExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "updater", "active"),
(Gauge<Integer>) () -> databaseUpdaterExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "notifier", "active"),
(Gauge<Integer>) () -> notifierExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "notifier", "queue"),
(Gauge<Integer>) () -> notifierExecutor.getQueue().size());
}
private void createExecutors() {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.notifierExecutor = newDiscardingExecutorService(config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
}
private void createExecutors() {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor =
newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.notifierExecutor =
newDiscardingExecutorService(
config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
}
public void start() {
createExecutors();
startFeedProcessingLoop();
startRefillLoop();
}
public void start() {
createExecutors();
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(
() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// send the feed to be processed
log.debug(
"got feed {} from the queue, send it for processing",
feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
// we removed a feed from the queue, try to refill it as it may now be
// empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(
() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
private void refillQueueAsync() {
CompletableFuture.runAsync(
() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug(
"found {} feeds that are up for refresh",
nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
},
refillExecutor)
.whenComplete(
(data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.thenCompose(r -> {
List<CompletableFuture<Void>> futures = r.insertedUnreadEntriesBySubscription().entrySet().stream().map(e -> {
FeedSubscription sub = e.getKey();
List<FeedEntry> entries = e.getValue();
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.thenCompose(
r -> {
List<CompletableFuture<Void>> futures =
r.insertedUnreadEntriesBySubscription().entrySet().stream()
.map(
e -> {
FeedSubscription sub = e.getKey();
List<FeedEntry> entries = e.getValue();
notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(() -> notifier.sendPushNotifications(sub, entries), notifierExecutor);
}).toList();
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new));
})
.exceptionally(ex -> {
log.error("error while processing feed {}", feed.getUrl(), ex);
return null;
});
}
notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(
() ->
notifier
.sendPushNotifications(
sub,
entries),
notifierExecutor);
})
.toList();
return CompletableFuture.allOf(
futures.toArray(CompletableFuture[]::new));
})
.exceptionally(
ex -> {
log.error("error while processing feed {}", feed.getUrl(), ex);
return null;
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
if (!feeds.isEmpty()) {
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
}
return feeds;
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(
() -> {
Instant lastLoginThreshold =
config.feedRefresh().userInactivityPeriod().isZero()
? null
: Instant.now()
.minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
if (!feeds.isEmpty()) {
// update disabledUntil to prevent feeds from being returned again by
// feedDAO.findNextUpdatable()
Instant nextUpdateDate =
Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(
feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
}
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(
this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
queue.clear();
}
queue.clear();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already
* running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that discards tasks if the queue is full
*/
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/** returns an ExecutorService that discards tasks if the queue is full */
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(
threads,
threads,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
/** returns an ExecutorService that blocks submissions until a thread is available */
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(
threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(
(r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}

139
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -1,84 +1,95 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import org.apache.commons.lang3.ObjectUtils;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public FeedRefreshIntervalCalculator(
CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFetchSuccess(
Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant =
empirical
? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
Duration retryInterval =
errorHandling
.backoffInterval()
.multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
private Instant computeEmpiricalRefreshInterval(
Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis =
Longs.constrainToRange(
averageEntryInterval / factor,
interval.toMillis(),
maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(
ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)),
instantSource.instant().plus(interval));
}
}

270
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -1,20 +1,5 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
@@ -29,142 +14,171 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.google.common.util.concurrent.Striped;
import jakarta.inject.Singleton;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
/**
* Updates the feed in the database and inserts new entries
*/
/** Updates the feed in the database and inserts new entries */
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final Striped<Lock> locks;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
public FeedRefreshUpdater(
UnitOfWork unitOfWork,
FeedService feedService,
FeedEntryService feedEntryService,
MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
locks = Striped.lazyWeakLock(100000);
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
FeedEntry insertedEntry = null;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
private AddEntryResult addEntry(
final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
FeedEntry insertedEntry = null;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
insertedEntry = unitOfWork.call(() -> {
if (feedEntryService.find(feed, entry) != null) {
// entry already exists, nothing to do
return null;
}
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
insertedEntry =
unitOfWork.call(
() -> {
if (feedEntryService.find(feed, entry) != null) {
// entry already exists, nothing to do
return null;
}
FeedEntry feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
return feedEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
Thread.currentThread().interrupt();
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
}
FeedEntry feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread =
feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
return feedEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error(
"interrupted while waiting for lock for {} : {}",
feed.getUrl(),
e.getMessage(),
e);
Thread.currentThread().interrupt();
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
}
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription =
new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
List<Entry> newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.insertedEntry != null ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> {
if (addEntryResult.insertedEntry != null) {
insertedUnreadEntriesBySubscription.computeIfAbsent(sub, k -> new ArrayList<>()).add(addEntryResult.insertedEntry);
}
});
}
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
List<Entry> newEntries =
unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.insertedEntry != null ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(
sub -> {
if (addEntryResult.insertedEntry != null) {
insertedUnreadEntriesBySubscription
.computeIfAbsent(sub, k -> new ArrayList<>())
.add(addEntryResult.insertedEntry);
}
});
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
unitOfWork.run(() -> feedService.update(feed));
return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
}
return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
}
private record AddEntryResult(boolean processed, FeedEntry insertedEntry, Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
public record FeedRefreshUpdaterResult(Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {}
private record AddEntryResult(
boolean processed,
FeedEntry insertedEntry,
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
public record FeedRefreshUpdaterResult(
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {}
}

190
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,15 +1,5 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.Strings;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
@@ -18,108 +8,132 @@ import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Strings;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link
* FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
public FeedRefreshWorker(
FeedRefreshIntervalCalculator refreshIntervalCalculator,
FeedFetcher fetcher,
CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result =
fetcher.fetch(
url,
false,
feed.getLastModifiedHeader(),
feed.getEtagHeader(),
feed.getLastPublishedDate(),
feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries =
entries.stream()
.filter(entry -> entry.published().isAfter(threshold))
.toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (Strings.CS.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
String urlAfterRedirect = result.urlAfterRedirect();
if (Strings.CS.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setIconUrl(result.feed().iconUrl());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setIconUrl(result.feed().iconUrl());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(
refreshIntervalCalculator.onFetchSuccess(
result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(),
result.validFor()));
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(
refreshIntervalCalculator.onFeedNotModified(
feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(
refreshIntervalCalculator.onTooManyRequests(
e.getRetryAfter(), feed.getErrorCount()));
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {}
}

55
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java
View File

@@ -1,9 +1,5 @@
package com.commafeed.backend.feed;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserSettingsDAO;
@@ -13,7 +9,8 @@ import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.PushNotificationService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import jakarta.inject.Singleton;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -22,29 +19,33 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class FeedUpdateNotifier {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions;
private final PushNotificationService pushNotificationService;
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions;
private final PushNotificationService pushNotificationService;
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) {
webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
}
}
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) {
webSocketSessions.sendMessage(
sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
}
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled() || !sub.isPushNotificationsEnabled() || entries.isEmpty()) {
return;
}
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null && settings.getPushNotifications() != null && settings.getPushNotifications().getType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
}
}
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled()
|| !sub.isPushNotificationsEnabled()
|| entries.isEmpty()) {
return;
}
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null
&& settings.getPushNotifications() != null
&& settings.getPushNotifications().getType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
}
}
}
}

136
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,14 +1,5 @@
package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.Date;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
@@ -16,87 +7,90 @@ import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import java.util.Collections;
import java.util.Date;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
* Utility methods related to feed handling
*
*/
/** Utility methods related to feed handling */
@UtilityClass
@Slf4j
public class FeedUtils {
public static String truncate(String string, int length) {
return StringUtils.truncate(string, length);
}
public static String truncate(String string, int length) {
return StringUtils.truncate(string, length);
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
return TextDirectionDetector.detect(stripped)
== TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + ImageProxyUrl.encode(url);
}
return "rest/server/proxy?u=" + ImageProxyUrl.encode(url);
}
public static SyndEntry asRss(Entry entry) {
SyndEntry e = new SyndEntryImpl();
public static SyndEntry asRss(Entry entry) {
SyndEntry e = new SyndEntryImpl();
e.setUri(entry.getGuid());
e.setTitle(entry.getTitle());
e.setAuthor(entry.getAuthor());
e.setUri(entry.getGuid());
e.setTitle(entry.getTitle());
e.setAuthor(entry.getAuthor());
SyndContentImpl c = new SyndContentImpl();
c.setValue(entry.getContent());
e.setContents(Collections.singletonList(c));
SyndContentImpl c = new SyndContentImpl();
c.setValue(entry.getContent());
e.setContents(Collections.singletonList(c));
if (entry.getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(entry.getEnclosureType());
enclosure.setUrl(entry.getEnclosureUrl());
e.setEnclosures(Collections.singletonList(enclosure));
}
e.setLink(entry.getUrl());
e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
return e;
}
if (entry.getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(entry.getEnclosureType());
enclosure.setUrl(entry.getEnclosureUrl());
e.setEnclosures(Collections.singletonList(enclosure));
}
e.setLink(entry.getUrl());
e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
return e;
}
}

83
commafeed-server/src/main/java/com/commafeed/backend/feed/ImageProxyUrl.java
View File

@@ -1,70 +1,65 @@
package com.commafeed.backend.feed;
import com.google.common.primitives.Bytes;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.lang3.RandomUtils;
import com.google.common.primitives.Bytes;
import lombok.experimental.UtilityClass;
import org.apache.commons.lang3.RandomUtils;
@UtilityClass
public class ImageProxyUrl {
private static final int GCM_IV_LENGTH = 12;
private static final int GCM_TAG_LENGTH = 128;
private static final int GCM_IV_LENGTH = 12;
private static final int GCM_TAG_LENGTH = 128;
private static SecretKey key;
private static SecretKey key;
public static void generateKey() {
key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES");
}
public static void generateKey() {
key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES");
}
public static String encode(String url) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
public static String encode(String url) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
try {
byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH);
try {
byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8));
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8));
byte[] combined = Bytes.concat(iv, encrypted);
return Base64.getUrlEncoder().withoutPadding().encodeToString(combined);
} catch (Exception e) {
throw new IllegalStateException("Failed to encode URL", e);
}
}
byte[] combined = Bytes.concat(iv, encrypted);
return Base64.getUrlEncoder().withoutPadding().encodeToString(combined);
} catch (Exception e) {
throw new IllegalStateException("Failed to encode URL", e);
}
}
public static String decode(String code) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
public static String decode(String code) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
try {
byte[] combined = Base64.getUrlDecoder().decode(code);
try {
byte[] combined = Base64.getUrlDecoder().decode(code);
byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH);
byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length);
byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH);
byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] decrypted = cipher.doFinal(encrypted);
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new IllegalStateException("Failed to decode URL", e);
}
}
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] decrypted = cipher.doFinal(encrypted);
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new IllegalStateException("Failed to decode URL", e);
}
}
}

105
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -1,70 +1,61 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.Strings;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import jakarta.inject.Singleton;
import java.nio.charset.Charset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.Strings;
@Singleton
public class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (Strings.CI.startsWith(extracted, "iso-8859-")) {
if (!Strings.CS.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (Strings.CI.startsWith(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by
* detecting the characters used in the feed
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (Strings.CI.startsWith(extracted, "iso-8859-")) {
if (!Strings.CS.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (Strings.CI.startsWith(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
/** Extract the declared encoding from the xml */
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = Strings.CS.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = Strings.CS.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
/** Detect encoding by analyzing characters in the array */
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

468
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -1,25 +1,5 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemProperties;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.Urls;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
@@ -38,258 +18,290 @@ import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemProperties;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
/**
* Parses raw xml into a FeedParserResult object
*/
/** Parses raw xml into a FeedParserResult object */
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Namespace ATOM_10_NS =
Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private static final Comparator<Entry> ENTRY_COMPARATOR = Comparator.comparing(Entry::published).reversed();
private static final Comparator<Entry> ENTRY_COMPARATOR =
Comparator.comparing(Entry::published).reversed();
private final EncodingDetector encodingDetector;
private final XMLCleaner xmlCleaner;
private final EncodingDetector encodingDetector;
private final XMLCleaner xmlCleaner;
public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) {
this.encodingDetector = encodingDetector;
this.xmlCleaner = xmlCleaner;
public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) {
this.encodingDetector = encodingDetector;
this.xmlCleaner = xmlCleaner;
// disable entity expansion limits added in JDK24+ (#1961)
// we already strip doctype declarations in XMLCleaner to prevent xxe attacks
// we also already limit the size of feeds we download in HttpGetter
System.setProperty(SystemProperties.JDK_XML_MAX_GENERAL_ENTITY_SIZE_LIMIT, "0");
System.setProperty(SystemProperties.JDK_XML_TOTAL_ENTITY_SIZE_LIMIT, "0");
}
// disable entity expansion limits added in JDK24+ (#1961)
// we already strip doctype declarations in XMLCleaner to prevent xxe attacks
// we also already limit the size of feeds we download in HttpGetter
System.setProperty(SystemProperties.JDK_XML_MAX_GENERAL_ENTITY_SIZE_LIMIT, "0");
System.setProperty(SystemProperties.JDK_XML_TOTAL_ENTITY_SIZE_LIMIT, "0");
}
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = xmlCleaner.clean(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is empty for url " + feedUrl);
}
String xmlString = xmlCleaner.clean(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is empty for url " + feedUrl);
}
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
String title = feed.getTitle();
String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null
|| lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
return new FeedParserResult(
title,
link,
iconUrl,
lastPublishedDate,
averageEntryInterval,
lastEntryDate,
entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(
String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
/** Adds atom links for rss feeds */
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(ENTRY_COMPARATOR);
return entries;
}
entries.sort(ENTRY_COMPARATOR);
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories =
StringUtils.trimToNull(
item.getCategories().stream()
.map(SyndCategory::getName)
.collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null);
if (enclosure == null) {
return null;
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || Urls.isAbsolute(url)) {
// url is absolute, nothing to do
return url;
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || Urls.isAbsolute(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return Urls.toAbsolute(url, feedLink, feedUrl);
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return Urls.toAbsolute(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content =
item.getContents().stream()
.map(SyndContent::getValue)
.collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff =
Math.abs(
entries.get(i).published().toEpochMilli()
- entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}

28
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -3,13 +3,29 @@ package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, String iconUrl, Instant lastPublishedDate, Long averageEntryInterval,
Instant lastEntryDate, List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record FeedParserResult(
String title,
String link,
String iconUrl,
Instant lastPublishedDate,
Long averageEntryInterval,
Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {}
public record Content(
String title,
String content,
String author,
String categories,
Enclosure enclosure,
Media media) {}
public record Enclosure(String url, String type) {}
public record Enclosure(String url, String type) {}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {}
public record Media(
String description,
String thumbnailUrl,
Integer thumbnailWidth,
Integer thumbnailHeight) {}
}

515
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
View File

@@ -4,268 +4,267 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final List<String> HTML_ENTITIES;
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final List<String> HTML_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = List.copyOf(map.keySet());
}
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = List.copyOf(map.keySet());
}
}

75
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -2,55 +2,54 @@ package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public enum Direction {
LEFT_TO_RIGHT,
RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}

107
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java
View File

@@ -1,10 +1,8 @@
package com.commafeed.backend.feed.parser;
import jakarta.inject.Singleton;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@@ -13,70 +11,71 @@ import org.jdom2.Verifier;
@Singleton
public class XMLCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private static final Pattern DOCTYPE_PATTERN =
Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private final Trie trie = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
private final Trie trie =
Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
public String clean(String xml) {
xml = removeCharactersBeforeFirstXmlTag(xml);
xml = removeInvalidXmlCharacters(xml);
xml = replaceHtmlEntitiesWithNumericEntities(xml);
xml = removeDoctypeDeclarations(xml);
return xml;
}
public String clean(String xml) {
xml = removeCharactersBeforeFirstXmlTag(xml);
xml = removeInvalidXmlCharacters(xml);
xml = replaceHtmlEntitiesWithNumericEntities(xml);
xml = removeDoctypeDeclarations(xml);
return xml;
}
String removeCharactersBeforeFirstXmlTag(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
String removeCharactersBeforeFirstXmlTag(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
int pos = xml.indexOf('<');
return pos < 0 ? null : xml.substring(pos);
}
int pos = xml.indexOf('<');
return pos < 0 ? null : xml.substring(pos);
}
String removeInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
String removeInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
return xml.codePoints()
.filter(Verifier::isXMLCharacter)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString();
}
return xml.codePoints()
.filter(Verifier::isXMLCharacter)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString();
}
// https://stackoverflow.com/a/40836618
String replaceHtmlEntitiesWithNumericEntities(String source) {
if (StringUtils.isBlank(source)) {
return null;
}
// https://stackoverflow.com/a/40836618
String replaceHtmlEntitiesWithNumericEntities(String source) {
if (StringUtils.isBlank(source)) {
return null;
}
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = trie.parseText(source);
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
return sb.toString();
}
String removeDoctypeDeclarations(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
String removeDoctypeDeclarations(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}

27
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
View File

@@ -1,33 +1,28 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator;
import java.io.Serializable;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
/** Abstract model for all entities, defining id and table generator */
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}

134
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import java.sql.Types;
import java.time.Instant;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "FEEDS")
@@ -20,97 +17,66 @@ import lombok.Setter;
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/** The url of the feed */
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
/** cache the url after potential http 30x redirects */
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
/** The url of the website, extracted from the feed */
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
@Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl;
@Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl;
/**
* Last time we tried to fetch the feed
*/
@Column
private Instant lastUpdated;
/** Last time we tried to fetch the feed */
@Column private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Column
private Instant lastPublishedDate;
/** Last publishedDate value in the feed */
@Column private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Column
private Instant lastEntryDate;
/** date of the last entry of the feed */
@Column private Instant lastEntryDate;
/**
* error message while retrieving the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/** error message while retrieving the feed */
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/** times we failed to retrieve the feed */
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Column
private Instant disabledUntil;
/** feed refresh is disabled until this date */
@Column private Instant disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/** http header returned by the feed */
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/** http header returned by the feed */
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
/** average time between entries in the feed */
private Long averageEntryInterval;
/** last hash of the content of the feed xml */
@Column(length = 40)
private String lastContentHash;
}

20
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
View File

@@ -6,7 +6,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@@ -17,18 +16,17 @@ import lombok.Setter;
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
private boolean collapsed;
private int position;
private boolean collapsed;
private int position;
}

53
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
View File

@@ -1,8 +1,5 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
@@ -11,7 +8,8 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.time.Instant;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
@@ -22,39 +20,32 @@ import lombok.Setter;
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
@Column(length = 2048)
private String url;
/**
* the moment the entry was inserted in the database
*/
@Column
private Instant inserted;
/** the moment the entry was inserted in the database */
@Column private Instant inserted;
/**
* the moment the entry was published in the feed
*
*/
@Column(name = "updated")
private Instant published;
/** the moment the entry was published in the feed */
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}

119
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
View File

@@ -1,21 +1,17 @@
package com.commafeed.backend.model;
import java.sql.Types;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.sql.Types;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@@ -24,73 +20,74 @@ import lombok.Setter;
@Setter
public class FeedEntryContent extends AbstractModel {
public enum Direction {
@JsonProperty("ltr")
LTR,
public enum Direction {
@JsonProperty("ltr")
LTR,
@JsonProperty("rtl")
RTL,
@JsonProperty("rtl")
RTL,
@JsonProperty("unknown")
UNKNOWN
}
@JsonProperty("unknown")
UNKNOWN
}
@Column(length = 2048)
private String title;
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Column(length = 40)
private String contentHash;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@Column(length = 4096)
private String categories;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.UNKNOWN;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.UNKNOWN;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
return new EqualsBuilder()
.append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
}

68
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
View File

@@ -1,9 +1,5 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
@@ -11,7 +7,9 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Transient;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
@@ -22,48 +20,40 @@ import lombok.Setter;
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Column(name = "read_status")
private boolean read;
@Transient
private boolean markable;
private boolean starred;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
@Transient private boolean markable;
/**
* Denormalization starts here
*/
@Transient private List<FeedEntryTag> tags = new ArrayList<>();
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
/** Denormalization starts here */
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Column
private Instant entryInserted;
@Column private Instant entryInserted;
@Column(name = "entryUpdated")
private Instant entryPublished;
@Column(name = "entryUpdated")
private Instant entryPublished;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
public FeedEntryStatus() {}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
}

31
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java
View File

@@ -6,7 +6,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@@ -17,24 +16,22 @@ import lombok.Setter;
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
public FeedEntryTag() {}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}

47
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java
View File

@@ -1,7 +1,5 @@
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
@@ -10,7 +8,7 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
@@ -21,35 +19,34 @@ import lombok.Setter;
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private int position;
private int position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
@Column(name = "filtering_expression", length = 4096)
private String filter;
@Column(name = "filtering_expression_legacy", length = 4096)
private String filterLegacy;
@Column(name = "filtering_expression_legacy", length = 4096)
private String filterLegacy;
@Column(name = "push_notifications_enabled")
private boolean pushNotificationsEnabled;
@Column(name = "auto_mark_as_read_after_days")
private Integer autoMarkAsReadAfterDays;
@Column(name = "push_notifications_enabled")
private boolean pushNotificationsEnabled;
@Column(name = "auto_mark_as_read_after_days")
private Integer autoMarkAsReadAfterDays;
}

47
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java
View File

@@ -2,40 +2,35 @@ package com.commafeed.backend.model;
import java.time.Duration;
import java.time.Instant;
import lombok.experimental.UtilityClass;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
public static final Instant MINIMUM_INSTANT =
Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/** initialize a proxy */
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
/** extract the id from the proxy without initializing it */
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}

57
commafeed-server/src/main/java/com/commafeed/backend/model/User.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import java.sql.Types;
import java.time.Instant;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "USERS")
@@ -20,40 +17,36 @@ import lombok.Setter;
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Column(length = 255, unique = true)
private String email;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Column(length = 40, unique = true)
private String apiKey;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Column(nullable = false)
private boolean disabled;
@Column
private Instant lastLogin;
@Column private Instant lastLogin;
@Column
private Instant created;
@Column private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Column(length = 40)
private String recoverPasswordToken;
@Column
private Instant recoverPasswordTokenDate;
@Column private Instant recoverPasswordTokenDate;
@Column
private Instant lastForceRefresh;
@Column private Instant lastForceRefresh;
}

33
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java
View File

@@ -8,7 +8,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@@ -19,25 +18,23 @@ import lombok.Setter;
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
public enum Role {
USER,
ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
public UserRole() {}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}

241
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java
View File

@@ -1,8 +1,6 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import java.sql.Types;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.persistence.Column;
import jakarta.persistence.Embeddable;
import jakarta.persistence.Embedded;
@@ -14,13 +12,11 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
import java.sql.Types;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "USERSETTINGS")
@@ -29,155 +25,154 @@ import lombok.Setter;
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
@JsonProperty("all")
ALL,
public enum ReadingMode {
@JsonProperty("all")
ALL,
@JsonProperty("unread")
UNREAD;
@JsonProperty("unread")
UNREAD;
// method called for query parameters
public static ReadingMode fromString(final String s) {
return ReadingMode.valueOf(s.toUpperCase());
}
}
// method called for query parameters
public static ReadingMode fromString(final String s) {
return ReadingMode.valueOf(s.toUpperCase());
}
}
public enum ReadingOrder {
@JsonProperty("asc")
ASC,
public enum ReadingOrder {
@JsonProperty("asc")
ASC,
@JsonProperty("desc")
DESC;
@JsonProperty("desc")
DESC;
// method called for query parameters
public static ReadingOrder fromString(final String s) {
return ReadingOrder.valueOf(s.toUpperCase());
}
}
// method called for query parameters
public static ReadingOrder fromString(final String s) {
return ReadingOrder.valueOf(s.toUpperCase());
}
}
public enum ScrollMode {
@JsonProperty("always")
ALWAYS,
public enum ScrollMode {
@JsonProperty("always")
ALWAYS,
@JsonProperty("never")
NEVER,
@JsonProperty("never")
NEVER,
@JsonProperty("if_needed")
IF_NEEDED
}
@JsonProperty("if_needed")
IF_NEEDED
}
public enum IconDisplayMode {
@JsonProperty("always")
ALWAYS,
public enum IconDisplayMode {
@JsonProperty("always")
ALWAYS,
@JsonProperty("never")
NEVER,
@JsonProperty("never")
NEVER,
@JsonProperty("on_desktop")
ON_DESKTOP,
@JsonProperty("on_desktop")
ON_DESKTOP,
@JsonProperty("on_mobile")
ON_MOBILE
}
@JsonProperty("on_mobile")
ON_MOBILE
}
public enum PushNotificationType {
@JsonProperty("ntfy")
NTFY,
public enum PushNotificationType {
@JsonProperty("ntfy")
NTFY,
@JsonProperty("gotify")
GOTIFY,
@JsonProperty("gotify")
GOTIFY,
@JsonProperty("pushover")
PUSHOVER
}
@JsonProperty("pushover")
PUSHOVER
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4)
private String language;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
private boolean showRead;
private boolean scrollMarks;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling;
private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
@Column(name = "primary_color", length = 32)
private String primaryColor;
@Column(name = "primary_color", length = 32)
private String primaryColor;
private boolean markAllAsReadConfirmation;
private boolean markAllAsReadNavigateToNextUnread;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean disablePullToRefresh;
private boolean markAllAsReadConfirmation;
private boolean markAllAsReadNavigateToNextUnread;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean disablePullToRefresh;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean instapaper;
private boolean buffer;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean instapaper;
private boolean buffer;
@Embedded
private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embedded
private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embeddable
@SuppressWarnings("serial")
@Getter
@Setter
public static class PushNotificationUserSettings implements Serializable {
@Enumerated(EnumType.STRING)
@Column(name = "push_notification_type", length = 16)
private PushNotificationType type;
@Embeddable
@SuppressWarnings("serial")
@Getter
@Setter
public static class PushNotificationUserSettings implements Serializable {
@Enumerated(EnumType.STRING)
@Column(name = "push_notification_type", length = 16)
private PushNotificationType type;
@Column(name = "push_notification_server_url", length = 1024)
private String serverUrl;
@Column(name = "push_notification_server_url", length = 1024)
private String serverUrl;
@Column(name = "push_notification_user_id", length = 512)
private String userId;
@Column(name = "push_notification_user_id", length = 512)
private String userId;
@Column(name = "push_notification_user_secret", length = 512)
private String userSecret;
@Column(name = "push_notification_topic", length = 256)
private String topic;
}
@Column(name = "push_notification_user_secret", length = 512)
private String userSecret;
@Column(name = "push_notification_topic", length = 256)
private String topic;
}
}

129
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
View File

@@ -1,13 +1,5 @@
package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
@@ -16,75 +8,88 @@ import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import jakarta.inject.Singleton;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.ObjectUtils;
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
private static final Comparator<FeedCategory> CATEGORY_COMPARATOR = Comparator
.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedSubscription> SUBSCRIPTION_COMPARATOR = Comparator
.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedCategory> CATEGORY_COMPARATOR =
Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedSubscription> SUBSCRIPTION_COMPARATOR =
Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(CATEGORY_COMPARATOR);
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(CATEGORY_COMPARATOR);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(SUBSCRIPTION_COMPARATOR);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(SUBSCRIPTION_COMPARATOR);
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
// export root subscriptions
for (FeedSubscription sub :
subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
return opml;
}
}
private Outline buildCategoryOutline(
FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child :
categories.stream()
.filter(
c ->
c.getParent() != null
&& c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub :
subscriptions.stream()
.filter(
s ->
s.getCategory() != null
&& s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}

121
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
View File

@@ -1,13 +1,5 @@
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.XMLCleaner;
@@ -18,71 +10,76 @@ import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {
private final XMLCleaner xmlCleaner;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final XMLCleaner xmlCleaner;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xmlCleaner.clean(xml);
if (xml == null) {
throw new IllegalArgumentException("Invalid OPML");
}
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xmlCleaner.clean(xml);
if (xml == null) {
throw new IllegalArgumentException("Invalid OPML");
}
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.persist(category);
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.persist(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(
user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
View File

@@ -1,38 +1,32 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
/**
* Support for OPML 1.1 parsing
*
*/
/** Support for OPML 1.1 parsing */
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
return e.getName().equals("opml");
return e.getName().equals("opml");
}
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale)
throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}

317
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -1,12 +1,14 @@
package com.commafeed.backend.service;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -21,155 +23,212 @@ import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES =
Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES =
Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] {'(', ')'};
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
clean.outputSettings(
new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags(
"a",
"b",
"blockquote",
"br",
"caption",
"cite",
"code",
"col",
"colgroup",
"dd",
"div",
"dl",
"dt",
"em",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"i",
"iframe",
"img",
"li",
"ol",
"p",
"pre",
"q",
"small",
"strike",
"strong",
"sub",
"sup",
"table",
"tbody",
"td",
"tfoot",
"th",
"thead",
"tr",
"u",
"ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes(
"iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes(
"td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes(
"th",
"border",
"bordercolor",
"abbr",
"axis",
"colspan",
"rowspan",
"scope",
"width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property)
&& StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
if (ALLOWED_IMG_CSS_RULES.contains(property)
&& StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
parser.setErrorHandler(
new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug(
"warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug(
"fatal error while parsing css: {}",
exception.getMessage(),
exception);
}
});
return parser;
}
return parser;
}
}

100
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -1,11 +1,5 @@
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
@@ -13,59 +7,65 @@ import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import jakarta.inject.Singleton;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.persist(entryContent);
return entryContent;
}
}
/** this is NOT thread-safe */
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing =
feedEntryContentDAO
.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.persist(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.RTL : FeedEntryContent.Direction.LTR);
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(
FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(
FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content())
? FeedEntryContent.Direction.RTL
: FeedEntryContent.Direction.LTR);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(
cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}

166
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -1,22 +1,7 @@
package com.commafeed.backend.service;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelValidationException;
import dev.cel.common.types.SimpleType;
@@ -25,83 +10,112 @@ import dev.cel.compiler.CelCompilerFactory;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory;
import jakarta.inject.Singleton;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final CelCompiler CEL_COMPILER = CelCompilerFactory.standardCelCompilerBuilder()
.addVar("title", SimpleType.STRING)
.addVar("titleLower", SimpleType.STRING)
.addVar("author", SimpleType.STRING)
.addVar("authorLower", SimpleType.STRING)
.addVar("content", SimpleType.STRING)
.addVar("contentLower", SimpleType.STRING)
.addVar("url", SimpleType.STRING)
.addVar("urlLower", SimpleType.STRING)
.addVar("categories", SimpleType.STRING)
.addVar("categoriesLower", SimpleType.STRING)
.build();
private static final CelRuntime CEL_RUNTIME = CelRuntimeFactory.standardCelRuntimeBuilder().build();
private static final CelCompiler CEL_COMPILER =
CelCompilerFactory.standardCelCompilerBuilder()
.addVar("title", SimpleType.STRING)
.addVar("titleLower", SimpleType.STRING)
.addVar("author", SimpleType.STRING)
.addVar("authorLower", SimpleType.STRING)
.addVar("content", SimpleType.STRING)
.addVar("contentLower", SimpleType.STRING)
.addVar("url", SimpleType.STRING)
.addVar("urlLower", SimpleType.STRING)
.addVar("categories", SimpleType.STRING)
.addVar("categoriesLower", SimpleType.STRING)
.build();
private static final CelRuntime CEL_RUNTIME =
CelRuntimeFactory.standardCelRuntimeBuilder().build();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry)
throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
String title = entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text();
String author = entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String content = entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text();
String url = entry.getUrl() == null ? "" : entry.getUrl();
String categories = entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories();
String title =
entry.getContent().getTitle() == null
? ""
: Jsoup.parse(entry.getContent().getTitle()).text();
String author =
entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String content =
entry.getContent().getContent() == null
? ""
: Jsoup.parse(entry.getContent().getContent()).text();
String url = entry.getUrl() == null ? "" : entry.getUrl();
String categories =
entry.getContent().getCategories() == null
? ""
: entry.getContent().getCategories();
Map<String, Object> data = new HashMap<>();
data.put("title", title);
data.put("titleLower", title.toLowerCase());
Map<String, Object> data = new HashMap<>();
data.put("title", title);
data.put("titleLower", title.toLowerCase());
data.put("author", author);
data.put("authorLower", author.toLowerCase());
data.put("author", author);
data.put("authorLower", author.toLowerCase());
data.put("content", content);
data.put("contentLower", content.toLowerCase());
data.put("content", content);
data.put("contentLower", content.toLowerCase());
data.put("url", url);
data.put("urlLower", url.toLowerCase());
data.put("url", url);
data.put("urlLower", url.toLowerCase());
data.put("categories", categories);
data.put("categoriesLower", categories.toLowerCase());
data.put("categories", categories);
data.put("categoriesLower", categories.toLowerCase());
Future<Object> future = executor.submit(() -> evaluateCelExpression(filter, data));
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
Future<Object> future = executor.submit(() -> evaluateCelExpression(filter, data));
Object result;
try {
result =
future.get(
config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(),
TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException(
"interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException(
"Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
return Boolean.TRUE.equals(result);
}
return Boolean.TRUE.equals(result);
}
private Object evaluateCelExpression(String expression, Map<String, Object> data)
throws CelValidationException, CelEvaluationException {
CelAbstractSyntaxTree ast = CEL_COMPILER.compile(expression).getAst();
CelRuntime.Program program = CEL_RUNTIME.createProgram(ast);
return program.eval(data);
}
private Object evaluateCelExpression(String expression, Map<String, Object> data)
throws CelValidationException, CelEvaluationException {
CelAbstractSyntaxTree ast = CEL_COMPILER.compile(expression).getAst();
CelRuntime.Program program = CEL_RUNTIME.createProgram(ast);
return program.eval(data);
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

228
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -1,12 +1,5 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
@@ -20,7 +13,11 @@ import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -29,113 +26,146 @@ import lombok.extern.slf4j.Slf4j;
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.persist(feedEntry);
return feedEntry;
}
feedEntryDAO.persist(feedEntry);
return feedEntry;
}
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList();
}
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes =
entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
return entries.stream()
.filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid())))
.toList();
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.persist(status);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.persist(status);
}
return matches;
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.merge(status);
}
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.merge(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.merge(status);
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.merge(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markSubscriptionEntries(
User user,
List<FeedSubscription> subscriptions,
Instant olderThan,
Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findBySubscriptions(
user,
subscriptions,
true,
keywords,
null,
-1,
-1,
null,
false,
null,
null,
null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
private void markList(
List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark =
statuses.stream()
.filter(FeedEntryStatus::isMarkable)
.filter(
s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null
|| entryDate == null
|| entryDate.isBefore(olderThan);
})
.filter(
s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null
|| insertedDate == null
|| insertedDate.isBefore(insertedBefore);
})
.toList();
statusesToMark.forEach(s -> {
s.setRead(true);
feedEntryStatusDAO.merge(s);
});
}
statusesToMark.forEach(
s -> {
s.setRead(true);
feedEntryStatusDAO.merge(s);
});
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -1,43 +1,42 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames =
existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
addList.forEach(feedEntryTagDAO::persist);
feedEntryTagDAO.delete(removeList);
}
List<FeedEntryTag> addList =
tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList =
existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
addList.forEach(feedEntryTagDAO::persist);
feedEntryTagDAO.delete(removeList);
}
}

99
commafeed-server/src/main/java/com/commafeed/backend/service/FeedFaviconService.java
View File

@@ -1,71 +1,72 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import org.apache.commons.lang3.ArrayUtils;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.favicon.FaviconFetcher;
import com.commafeed.backend.model.Feed;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
@Singleton
@Slf4j
public class FeedFaviconService {
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST = Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST =
Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
}
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers;
this.defaultFavicon =
new Favicon(
Resources.toByteArray(
Objects.requireNonNull(
getClass().getResource("/images/default_favicon.gif"))),
"image/gif");
}
public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) {
return icon;
}
}
return defaultFavicon;
}
public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) {
return icon;
}
}
return defaultFavicon;
}
private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false;
}
private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false;
}
long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false;
}
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false;
}
return true;
}
return true;
}
}

54
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,46 +1,42 @@
package com.commafeed.backend.service;
import java.time.Instant;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import jakarta.inject.Singleton;
import java.time.Instant;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class FeedService {
private final FeedDAO feedDAO;
private final FeedDAO feedDAO;
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = Urls.normalize(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = Urls.normalize(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
}

193
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -1,12 +1,5 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
@@ -18,107 +11,125 @@ import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO, FeedService feedService,
FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
public FeedSubscriptionService(
FeedEntryStatusDAO feedEntryStatusDAO,
FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService,
FeedRefreshEngine feedRefreshEngine,
CommaFeedConfiguration config) {
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being
// processed
// by the feed engine
feedSubscriptionDAO.onPostCommitInsert(
sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null
|| feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
public long subscribe(
User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message =
String.format(
"You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (Urls.isHttp(feed.getUrl()) && Urls.isHttps(url)) {
feed.setUrl(url);
}
// upgrade feed to https if it was using http
if (Urls.isHttp(feed.getUrl()) && Urls.isHttps(url)) {
feed.setUrl(url);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
return feedSubscriptionDAO.merge(sub).getId();
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
return feedSubscriptionDAO.merge(sub).getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null
&& lastForceRefresh
.plus(config.feedRefresh().forceRefreshCooldownDuration())
.isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
user.setLastForceRefresh(Instant.now());
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user).stream()
.collect(
Collectors.toMap(
FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}

14
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java
View File

@@ -1,21 +1,19 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}

127
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java
View File

@@ -1,20 +1,16 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@Slf4j
@@ -22,72 +18,71 @@ import lombok.extern.slf4j.Slf4j;
@Singleton
public class PasswordEncryptionService {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

307
commafeed-server/src/main/java/com/commafeed/backend/service/PushNotificationService.java
View File

@@ -1,12 +1,20 @@
package com.commafeed.backend.service;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpClientFactory;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings.PushNotificationUserSettings;
import io.vertx.core.json.JsonObject;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig;
@@ -18,163 +26,196 @@ import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.util.Timeout;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpClientFactory;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings.PushNotificationUserSettings;
import io.vertx.core.json.JsonObject;
import lombok.extern.slf4j.Slf4j;
@Singleton
@Slf4j
public class PushNotificationService {
private final CloseableHttpClient httpClient;
private final Meter meter;
private final CommaFeedConfiguration config;
private final CloseableHttpClient httpClient;
private final Meter meter;
private final CommaFeedConfiguration config;
public PushNotificationService(HttpClientFactory httpClientFactory, MetricRegistry metrics, CommaFeedConfiguration config) {
this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads());
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify"));
this.config = config;
}
public PushNotificationService(
HttpClientFactory httpClientFactory,
MetricRegistry metrics,
CommaFeedConfiguration config) {
this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads());
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify"));
this.config = config;
}
public void notify(PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
if (!config.pushNotifications().enabled() || settings.getType() == null) {
return;
}
public void notify(
PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
if (!config.pushNotifications().enabled() || settings.getType() == null) {
return;
}
log.debug("sending {} push notification for entry {} in feed {}", settings.getType(), entry.getId(),
subscription.getFeed().getId());
String entryTitle = entry.getContent() != null ? entry.getContent().getTitle() : null;
String entryUrl = entry.getUrl();
String feedTitle = subscription.getTitle();
log.debug(
"sending {} push notification for entry {} in feed {}",
settings.getType(),
entry.getId(),
subscription.getFeed().getId());
String entryTitle = entry.getContent() != null ? entry.getContent().getTitle() : null;
String entryUrl = entry.getUrl();
String feedTitle = subscription.getTitle();
if (StringUtils.isBlank(entryTitle)) {
entryTitle = "New entry";
}
if (StringUtils.isBlank(entryTitle)) {
entryTitle = "New entry";
}
try {
switch (settings.getType()) {
case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl);
case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl);
case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl);
default -> throw new IllegalStateException("unsupported notification type: " + settings.getType());
}
} catch (IOException e) {
throw new PushNotificationException("Failed to send external notification", e);
}
try {
switch (settings.getType()) {
case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl);
case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl);
case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl);
default ->
throw new IllegalStateException(
"unsupported notification type: " + settings.getType());
}
} catch (IOException e) {
throw new PushNotificationException("Failed to send external notification", e);
}
meter.mark();
}
meter.mark();
}
private void sendNtfy(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl) throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String topic = settings.getTopic();
private void sendNtfy(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String topic = settings.getTopic();
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(topic)) {
log.warn("ntfy notification skipped: missing server URL or topic");
return;
}
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(topic)) {
log.warn("ntfy notification skipped: missing server URL or topic");
return;
}
HttpPost request = new HttpPost(serverUrl + "/" + topic);
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build());
request.addHeader("Title", feedTitle);
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8));
HttpPost request = new HttpPost(serverUrl + "/" + topic);
request.setConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.addHeader("Title", feedTitle);
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(entryUrl)) {
request.addHeader("Click", entryUrl);
}
if (StringUtils.isNotBlank(entryUrl)) {
request.addHeader("Click", entryUrl);
}
if (StringUtils.isNotBlank(settings.getUserSecret())) {
request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
}
if (StringUtils.isNotBlank(settings.getUserSecret())) {
request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
}
httpClient.execute(request, response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException("ntfy notification failed with status " + response.getCode());
}
return null;
});
}
httpClient.execute(
request,
response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException(
"ntfy notification failed with status " + response.getCode());
}
return null;
});
}
private void sendGotify(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String token = settings.getUserSecret();
private void sendGotify(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String token = settings.getUserSecret();
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(token)) {
log.warn("gotify notification skipped: missing server URL or token");
return;
}
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(token)) {
log.warn("gotify notification skipped: missing server URL or token");
return;
}
JsonObject json = new JsonObject();
json.put("title", feedTitle);
json.put("message", entryTitle);
json.put("priority", 5);
if (StringUtils.isNotBlank(entryUrl)) {
json.put("extras",
new JsonObject().put("client::notification", new JsonObject().put("click", new JsonObject().put("url", entryUrl))));
}
JsonObject json = new JsonObject();
json.put("title", feedTitle);
json.put("message", entryTitle);
json.put("priority", 5);
if (StringUtils.isNotBlank(entryUrl)) {
json.put(
"extras",
new JsonObject()
.put(
"client::notification",
new JsonObject()
.put("click", new JsonObject().put("url", entryUrl))));
}
HttpPost request = new HttpPost(serverUrl + "/message");
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build());
request.addHeader("X-Gotify-Key", token);
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON));
HttpPost request = new HttpPost(serverUrl + "/message");
request.setConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.addHeader("X-Gotify-Key", token);
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON));
httpClient.execute(request, response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException("gotify notification failed with status " + response.getCode());
}
return null;
});
}
httpClient.execute(
request,
response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException(
"gotify notification failed with status " + response.getCode());
}
return null;
});
}
private void sendPushover(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String token = settings.getUserSecret();
String userKey = settings.getUserId();
private void sendPushover(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String token = settings.getUserSecret();
String userKey = settings.getUserId();
if (StringUtils.isBlank(token) || StringUtils.isBlank(userKey)) {
log.warn("pushover notification skipped: missing token or user key");
return;
}
if (StringUtils.isBlank(token) || StringUtils.isBlank(userKey)) {
log.warn("pushover notification skipped: missing token or user key");
return;
}
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("token", token));
params.add(new BasicNameValuePair("user", userKey));
params.add(new BasicNameValuePair("title", feedTitle));
params.add(new BasicNameValuePair("message", entryTitle));
if (StringUtils.isNotBlank(entryUrl)) {
params.add(new BasicNameValuePair("url", entryUrl));
}
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("token", token));
params.add(new BasicNameValuePair("user", userKey));
params.add(new BasicNameValuePair("title", feedTitle));
params.add(new BasicNameValuePair("message", entryTitle));
if (StringUtils.isNotBlank(entryUrl)) {
params.add(new BasicNameValuePair("url", entryUrl));
}
HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json");
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build());
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json");
request.setConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
httpClient.execute(request, response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException("pushover notification failed with status " + response.getCode());
}
return null;
});
}
httpClient.execute(
request,
response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException(
"pushover notification failed with status " + response.getCode());
}
return null;
});
}
public static class PushNotificationException extends RuntimeException {
private static final long serialVersionUID = -3392881821584833819L;
public static class PushNotificationException extends RuntimeException {
private static final long serialVersionUID = -3392881821584833819L;
public PushNotificationException(String message) {
super(message);
}
public PushNotificationException(String message) {
super(message);
}
public PushNotificationException(String message, Throwable cause) {
super(message, cause);
}
}
public PushNotificationException(String message, Throwable cause) {
super(message, cause);
}
}
}

241
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConstants;
import com.commafeed.backend.Digests;
@@ -24,138 +13,152 @@ import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
/** try to log in with given credentials */
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated =
encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
/** try to log in with given api key */
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
/** try to log in with given fever api key */
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equalsIgnoreCase(feverApiKey)) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equalsIgnoreCase(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
/** should triggers after successful login */
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
public User register(
String name,
String password,
String email,
Collection<Role> roles,
boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
if (!forceRegistration) {
Preconditions.checkState(
config.users().allowRegistrations(),
"Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.persist(user);
for (Role role : roles) {
userRoleDAO.persist(new UserRole(user, role));
}
return user;
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.persist(user);
for (Role role : roles) {
userRoleDAO.persist(new UserRole(user, role));
}
return user;
}
public void createDemoUser() {
register(CommaFeedConstants.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void createDemoUser() {
register(
CommaFeedConstants.USERNAME_DEMO,
"demo",
"demo@commafeed.com",
Collections.singletonList(Role.USER),
true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public String generateApiKey(User user) {
byte[] key =
encryptionService.getEncryptedPassword(
UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

328
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -1,10 +1,5 @@
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
@@ -16,173 +11,202 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
/** Contains utility methods for cleaning the database */
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final int batchSize;
private final boolean keepStarredEntries;
private final Meter entriesDeletedMeter;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final int batchSize;
private final boolean keepStarredEntries;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.keepStarredEntries = config.database().cleanup().keepStarredEntries();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public DatabaseCleaningService(
CommaFeedConfiguration config,
UnitOfWork unitOfWork,
FeedDAO feedDAO,
FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO,
FeedEntryStatusDAO feedEntryStatusDAO,
MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.keepStarredEntries = config.database().cleanup().keepStarredEntries();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
entriesDeleted =
unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted =
unitOfWork.call(
() ->
feedDAO.delete(
feedDAO.findByIds(
feeds.stream()
.map(AbstractModel::getId)
.toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of contents without entries");
return;
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of contents without entries");
return;
}
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
List<FeedCapacity> feeds = unitOfWork
.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize, keepStarredEntries));
if (feeds.isEmpty()) {
break;
}
List<FeedCapacity> feeds =
unitOfWork.call(
() ->
feedEntryDAO.findFeedsExceedingCapacity(
maxFeedCapacity, batchSize, keepStarredEntries));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.capacity() - maxFeedCapacity;
int deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.capacity() - maxFeedCapacity;
int deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info(
"interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
final long rem = remaining;
deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.id(), Math.min(batchSize, rem), keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (deleted > 0 && remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
final long rem = remaining;
deleted =
unitOfWork.call(
() ->
feedEntryDAO.deleteOldEntries(
feed.id(),
Math.min(batchSize, rem),
keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (deleted > 0 && remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old entries");
return;
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old entries");
return;
}
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize, keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
deleted =
unitOfWork.call(
() ->
feedEntryDAO.deleteEntriesOlderThan(
olderThan, batchSize, keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old read statuses");
return;
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old read statuses");
return;
}
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
deleted =
unitOfWork.call(
() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
public void autoMarkAsRead() {
log.info("marking entries as read based on autoMarkAsReadAfterDays");
long total = 0;
long marked;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping marking entries as read");
return;
}
public void autoMarkAsRead() {
log.info("marking entries as read based on autoMarkAsReadAfterDays");
long total = 0;
long marked;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping marking entries as read");
return;
}
marked = unitOfWork.call(() -> feedEntryStatusDAO.autoMarkAsRead(batchSize));
total += marked;
log.debug("marked {} entries as read", total);
} while (marked != 0);
log.info("cleanup done: marked {} entries as read", total);
}
marked = unitOfWork.call(() -> feedEntryStatusDAO.autoMarkAsRead(batchSize));
total += marked;
log.debug("marked {} entries as read", total);
} while (marked != 0);
log.info("cleanup done: marked {} entries as read", total);
}
}

48
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -1,44 +1,42 @@
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import jakarta.inject.Singleton;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.kohsuke.MetaInfServices;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
public boolean isInitialSetupRequired() {
return unitOfWork.call(userDAO::count) == 0;
}
public boolean isInitialSetupRequired() {
return unitOfWork.call(userDAO::count) == 0;
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use
* lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(
String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java
View File

@@ -1,30 +1,29 @@
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.merge(user));
}
}
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache
// every
// time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.merge(user));
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/task/AutoMarkAsReadTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class AutoMarkAsReadTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.autoMarkAsRead();
}
@Override
public void run() {
cleaner.autoMarkAsRead();
}
@Override
public long getInitialDelay() {
return 30;
}
@Override
public long getInitialDelay() {
return 30;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

72
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java
View File

@@ -1,16 +1,13 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConstants;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -19,43 +16,42 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(() -> {
User demoUser = userDAO.findByName(CommaFeedConstants.USERNAME_DEMO);
if (demoUser == null) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(
() -> {
User demoUser = userDAO.findByName(CommaFeedConstants.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
userService.unregister(demoUser);
userService.createDemoUser();
});
}
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

50
commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java
View File

@@ -1,42 +1,38 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

56
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java
View File

@@ -1,45 +1,41 @@
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

52
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java
View File

@@ -1,43 +1,39 @@
package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java
View File

@@ -2,29 +2,34 @@ package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.debug("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(),
getTimeUnit(), getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
public void register(ScheduledExecutorService executor) {
Runnable runnable =
() -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.debug(
"registering task {} for execution every {} {}, starting in {} {}",
getClass().getSimpleName(),
getPeriod(),
getTimeUnit(),
getInitialDelay(),
getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/task/TaskScheduler.java
View File

@@ -1,37 +1,34 @@
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.google.common.util.concurrent.MoreExecutors;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final CommaFeedConfiguration config;
private final List<ScheduledTask> tasks;
private final CommaFeedConfiguration config;
private ScheduledExecutorService executor;
private ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks, CommaFeedConfiguration config) {
this.tasks = tasks;
this.config = config;
}
public TaskScheduler(@All List<ScheduledTask> tasks, CommaFeedConfiguration config) {
this.tasks = tasks;
this.config = config;
}
public void start() {
this.executor = Executors.newScheduledThreadPool(tasks.size());
this.tasks.forEach(task -> task.register(executor));
}
public void start() {
this.executor = Executors.newScheduledThreadPool(tasks.size());
this.tasks.forEach(task -> task.register(executor));
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(executor, config.shutdownTimeout());
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(executor, config.shutdownTimeout());
}
}

7
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java
View File

@@ -2,11 +2,8 @@ package com.commafeed.backend.urlprovider;
import java.util.List;
/**
* Tries to find a feed url given the url and page content
*/
/** Tries to find a feed url given the url and page content */
public interface FeedURLProvider {
List<String> get(String url, String urlContent);
List<String> get(String url, String urlContent);
}

27
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java
View File

@@ -1,25 +1,24 @@
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.stream.Stream;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public List<String> get(String url, String urlContent) {
Document doc = Jsoup.parse(urlContent, url);
if (!"html".equals(doc.children().getFirst().tagName())) {
return List.of();
}
return Stream.concat(doc.select("link[type=application/atom+xml]").stream(), doc.select("link[type=application/rss+xml]").stream())
.map(node -> node.attr("abs:href"))
.toList();
}
@Override
public List<String> get(String url, String urlContent) {
Document doc = Jsoup.parse(urlContent, url);
if (!"html".equals(doc.children().getFirst().tagName())) {
return List.of();
}
return Stream.concat(
doc.select("link[type=application/atom+xml]").stream(),
doc.select("link[type=application/rss+xml]").stream())
.map(node -> node.attr("abs:href"))
.toList();
}
}

28
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java
View File

@@ -1,30 +1,28 @@
package com.commafeed.backend.urlprovider;
import java.util.List;
import jakarta.inject.Singleton;
import java.util.List;
import org.apache.commons.lang3.Strings;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
*
* <p>converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public List<String> get(String url, String urlContent) {
if (!Strings.CI.startsWith(url, PREFIX)) {
return List.of();
}
return List.of(REPLACEMENT_PREFIX + url.substring(PREFIX.length()));
}
@Override
public List<String> get(String url, String urlContent) {
if (!Strings.CI.startsWith(url, PREFIX)) {
return List.of();
}
return List.of(REPLACEMENT_PREFIX + url.substring(PREFIX.length()));
}
}

40
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java
View File

@@ -1,13 +1,11 @@
package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@@ -15,27 +13,27 @@ import lombok.Data;
@RegisterForReflection
public class Category implements Serializable {
@Schema(description = "category id", required = true)
private String id;
@Schema(description = "category id", required = true)
private String id;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "category id", required = true)
private String name;
@Schema(description = "category id", required = true)
private String name;
@Schema(description = "category children categories", required = true)
private List<Category> children = new ArrayList<>();
@Schema(description = "category children categories", required = true)
private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", required = true)
private boolean expanded;
@Schema(description = "whether the category is expanded or collapsed", required = true)
private boolean expanded;
@Schema(description = "position of the category in the list", required = true)
private int position;
}
@Schema(description = "position of the category in the list", required = true)
private int position;
}

54
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java
View File

@@ -1,13 +1,11 @@
package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata")
@@ -15,36 +13,38 @@ import lombok.Data;
@RegisterForReflection
public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", required = true)
private String name;
@Schema(description = "name of the feed or the category requested", required = true)
private String name;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "times the server tried to refresh the feed and failed", required = true)
private int errorCount;
@Schema(description = "times the server tried to refresh the feed and failed", required = true)
private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(
description =
"URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", required = true)
private long timestamp;
@Schema(description = "list generation timestamp", required = true)
private long timestamp;
@Schema(description = "if the query has more elements", required = true)
private boolean hasMore;
@Schema(description = "if the query has more elements", required = true)
private boolean hasMore;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
@Schema(description = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>();
@Schema(
description =
"if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
}

200
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java
View File

@@ -1,22 +1,19 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.Strings;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import lombok.Data;
import org.apache.commons.lang3.Strings;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@@ -24,123 +21,134 @@ import lombok.Data;
@RegisterForReflection
public class Entry implements Serializable {
@Schema(description = "entry id", required = true)
private String id;
@Schema(description = "entry id", required = true)
private String id;
@Schema(description = "entry guid", required = true)
private String guid;
@Schema(description = "entry guid", required = true)
private String guid;
@Schema(description = "entry title", required = true)
private String title;
@Schema(description = "entry title", required = true)
private String title;
@Schema(description = "entry content", required = true)
private String content;
@Schema(description = "entry content", required = true)
private String content;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "whether entry content and title are rtl", required = true)
private boolean rtl;
@Schema(description = "whether entry content and title are rtl", required = true)
private boolean rtl;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = SchemaType.INTEGER, required = true)
private Instant date;
@Schema(description = "entry publication date", type = SchemaType.INTEGER, required = true)
private Instant date;
@Schema(description = "entry insertion date in the database", type = SchemaType.INTEGER, required = true)
private Instant insertedDate;
@Schema(
description = "entry insertion date in the database",
type = SchemaType.INTEGER,
required = true)
private Instant insertedDate;
@Schema(description = "feed id", required = true)
private String feedId;
@Schema(description = "feed id", required = true)
private String feedId;
@Schema(description = "feed name", required = true)
private String feedName;
@Schema(description = "feed name", required = true)
private String feedName;
@Schema(description = "this entry's feed url", required = true)
private String feedUrl;
@Schema(description = "this entry's feed url", required = true)
private String feedUrl;
@Schema(description = "this entry's website url", required = true)
private String feedLink;
@Schema(description = "this entry's website url", required = true)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "entry url", required = true)
private String url;
@Schema(description = "entry url", required = true)
private String url;
@Schema(description = "read status", required = true)
private boolean read;
@Schema(description = "read status", required = true)
private boolean read;
@Schema(description = "starred status", required = true)
private boolean starred;
@Schema(description = "starred status", required = true)
private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", required = true)
private boolean markable;
@Schema(
description = "whether the entry is still markable (old entry statuses are discarded)",
required = true)
private boolean markable;
@Schema(description = "tags", required = true)
private List<String> tags;
@Schema(description = "tags", required = true)
private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) {
entry.setRtl(content.getDirection() == FeedEntryContent.Direction.RTL);
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent());
entry.setAuthor(content.getAuthor());
if (content != null) {
entry.setRtl(content.getDirection() == FeedEntryContent.Direction.RTL);
entry.setTitle(content.getTitle());
entry.setContent(
proxyImages
? FeedUtils.proxyImages(content.getContent())
: content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && Strings.CS.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(
proxyImages && Strings.CS.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(
proxyImages
? FeedUtils.proxyImage(content.getMediaThumbnailUrl())
: content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
entry.setCategories(content.getCategories());
}
return entry;
}
}

15
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java
View File

@@ -1,11 +1,9 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Feed details")
@@ -13,10 +11,9 @@ import lombok.Data;
@RegisterForReflection
public class FeedInfo implements Serializable {
@Schema(description = "url", required = true)
private String url;
@Schema(description = "title", required = true)
private String title;
@Schema(description = "url", required = true)
private String url;
@Schema(description = "title", required = true)
private String title;
}

62
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java
View File

@@ -1,11 +1,9 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Server infos")
@@ -13,46 +11,44 @@ import lombok.Data;
@RegisterForReflection
public class ServerInfo implements Serializable {
@Schema
private String announcement;
@Schema private String announcement;
@Schema(required = true)
private String version;
@Schema(required = true)
private String version;
@Schema(required = true)
private String gitCommit;
@Schema(required = true)
private String gitCommit;
@Schema(required = true)
private boolean allowRegistrations;
@Schema(required = true)
private boolean allowRegistrations;
@Schema(required = true)
private boolean emailAddressRequired;
@Schema(required = true)
private boolean emailAddressRequired;
@Schema(required = true)
private boolean smtpEnabled;
@Schema(required = true)
private boolean smtpEnabled;
@Schema(required = true)
private boolean demoAccountEnabled;
@Schema(required = true)
private boolean demoAccountEnabled;
@Schema(required = true)
private boolean websocketEnabled;
@Schema(required = true)
private boolean websocketEnabled;
@Schema(required = true)
private long websocketPingInterval;
@Schema(required = true)
private long websocketPingInterval;
@Schema(required = true)
private long treeReloadInterval;
@Schema(required = true)
private long treeReloadInterval;
@Schema(required = true)
private long forceRefreshCooldownDuration;
@Schema(required = true)
private long forceRefreshCooldownDuration;
@Schema(required = true)
private boolean initialSetupRequired;
@Schema(required = true)
private boolean initialSetupRequired;
@Schema(required = true)
private int minimumPasswordLength;
@Schema(required = true)
private boolean pushNotificationsEnabled;
@Schema(required = true)
private int minimumPasswordLength;
@Schema(required = true)
private boolean pushNotificationsEnabled;
}

174
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.model.UserSettings.IconDisplayMode;
import com.commafeed.backend.model.UserSettings.PushNotificationType;
import com.commafeed.backend.model.UserSettings.ReadingMode;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.backend.model.UserSettings.ScrollMode;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "User settings")
@@ -19,113 +16,128 @@ import lombok.Data;
@RegisterForReflection
public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none")
private String language;
@Schema(description = "user's preferred language, english if none")
private String language;
@Schema(description = "user reads all entries or unread entries only", required = true)
private ReadingMode readingMode;
@Schema(description = "user reads all entries or unread entries only", required = true)
private ReadingMode readingMode;
@Schema(description = "user reads entries in ascending or descending order", required = true)
private ReadingOrder readingOrder;
@Schema(description = "user reads entries in ascending or descending order", required = true)
private ReadingOrder readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", required = true)
private boolean showRead;
@Schema(
description = "user wants category and feeds with no unread entries shown",
required = true)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", required = true)
private boolean scrollMarks;
@Schema(
description = "In expanded view, scroll through entries mark them as read",
required = true)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", required = true)
private int scrollSpeed;
@Schema(
description = "user's preferred scroll speed when navigating between entries",
required = true)
private int scrollSpeed;
@Schema(description = "whether to scroll to the selected entry", required = true)
private ScrollMode scrollMode;
@Schema(description = "whether to scroll to the selected entry", required = true)
private ScrollMode scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", required = true)
private int entriesToKeepOnTopWhenScrolling;
@Schema(
description = "number of entries to keep above the selected entry when scrolling",
required = true)
private int entriesToKeepOnTopWhenScrolling;
@Schema(description = "whether to show the star icon in the header of entries", required = true)
private IconDisplayMode starIconDisplayMode;
@Schema(description = "whether to show the star icon in the header of entries", required = true)
private IconDisplayMode starIconDisplayMode;
@Schema(description = "whether to show the external link icon in the header of entries", required = true)
private IconDisplayMode externalLinkIconDisplayMode;
@Schema(
description = "whether to show the external link icon in the header of entries",
required = true)
private IconDisplayMode externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", required = true)
private boolean markAllAsReadConfirmation;
@Schema(description = "ask for confirmation when marking all entries as read", required = true)
private boolean markAllAsReadConfirmation;
@Schema(description = "navigate to the next unread category or feed after marking all entries as read", required = true)
private boolean markAllAsReadNavigateToNextUnread;
@Schema(
description =
"navigate to the next unread category or feed after marking all entries as read",
required = true)
private boolean markAllAsReadNavigateToNextUnread;
@Schema(description = "show commafeed's own context menu on right click", required = true)
private boolean customContextMenu;
@Schema(description = "show commafeed's own context menu on right click", required = true)
private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", required = true)
private boolean mobileFooter;
@Schema(
description = "on mobile, show action buttons at the bottom of the screen",
required = true)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", required = true)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the title", required = true)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", required = true)
private boolean unreadCountFavicon;
@Schema(description = "show unread count in the favicon", required = true)
private boolean unreadCountFavicon;
@Schema(description = "disable pull to refresh", required = true)
private boolean disablePullToRefresh;
@Schema(description = "disable pull to refresh", required = true)
private boolean disablePullToRefresh;
@Schema(description = "primary theme color to use in the UI")
private String primaryColor;
@Schema(description = "primary theme color to use in the UI")
private String primaryColor;
@Schema(description = "sharing settings", required = true)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "sharing settings", required = true)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "push notification settings", required = true)
private PushNotificationSettings pushNotificationSettings = new PushNotificationSettings();
@Schema(description = "push notification settings", required = true)
private PushNotificationSettings pushNotificationSettings = new PushNotificationSettings();
@Schema(description = "User notification settings")
@Data
public static class PushNotificationSettings implements Serializable {
@Schema(description = "notification provider type")
private PushNotificationType type;
@Schema(description = "User notification settings")
@Data
public static class PushNotificationSettings implements Serializable {
@Schema(description = "notification provider type")
private PushNotificationType type;
@Schema(description = "server URL for ntfy or gotify")
private String serverUrl;
@Schema(description = "server URL for ntfy or gotify")
private String serverUrl;
@Schema(description = "user Id")
private String userId;
@Schema(description = "user Id")
private String userId;
@Schema(description = "user secret for authentication with the service")
private String userSecret;
@Schema(description = "user secret for authentication with the service")
private String userSecret;
@Schema(description = "topic")
private String topic;
}
@Schema(description = "topic")
private String topic;
}
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(required = true)
private boolean email;
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(required = true)
private boolean email;
@Schema(required = true)
private boolean gmail;
@Schema(required = true)
private boolean gmail;
@Schema(required = true)
private boolean facebook;
@Schema(required = true)
private boolean facebook;
@Schema(required = true)
private boolean twitter;
@Schema(required = true)
private boolean twitter;
@Schema(required = true)
private boolean tumblr;
@Schema(required = true)
private boolean tumblr;
@Schema(required = true)
private boolean instapaper;
@Schema(required = true)
private boolean instapaper;
@Schema(required = true)
private boolean buffer;
}
@Schema(required = true)
private boolean buffer;
}
}

141
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java
View File

@@ -1,18 +1,15 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "User information")
@@ -20,80 +17,90 @@ import lombok.Data;
@RegisterForReflection
public class Subscription implements Serializable {
@Schema(description = "subscription id", required = true)
private Long id;
@Schema(description = "subscription id", required = true)
private Long id;
@Schema(description = "subscription name", required = true)
private String name;
@Schema(description = "subscription name", required = true)
private String name;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error count", required = true)
private int errorCount;
@Schema(description = "error count", required = true)
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = SchemaType.INTEGER)
private Instant lastRefresh;
@Schema(description = "last time the feed was refreshed", type = SchemaType.INTEGER)
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = SchemaType.INTEGER)
private Instant nextRefresh;
@Schema(
description =
"next time the feed refresh is planned, null if refresh is already queued",
type = SchemaType.INTEGER)
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", required = true)
private String feedUrl;
@Schema(description = "this subscription's feed url", required = true)
private String feedUrl;
@Schema(description = "this subscription's website url", required = true)
private String feedLink;
@Schema(description = "this subscription's website url", required = true)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "unread count", required = true)
private long unread;
@Schema(description = "unread count", required = true)
private long unread;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "date of the newest item", type = SchemaType.INTEGER)
private Instant newestItemTime;
@Schema(description = "date of the newest item", type = SchemaType.INTEGER)
private Instant newestItemTime;
@Schema(description = "CEL string evaluated on new entries to mark them as read if they do not match")
private String filter;
@Schema(
description =
"CEL string evaluated on new entries to mark them as read if they do not match")
private String filter;
@Schema(description = "JEXL legacy filter")
private String filterLegacy;
@Schema(description = "JEXL legacy filter")
private String filterLegacy;
@Schema(description = "whether to send push notifications for new entries of this feed", required = true)
private boolean pushNotificationsEnabled;
@Schema(
description = "whether to send push notifications for new entries of this feed",
required = true)
private boolean pushNotificationsEnabled;
@Schema(description = "automatically mark entries as read after this many days (null to disable)")
private Integer autoMarkAsReadAfterDays;
@Schema(
description =
"automatically mark entries as read after this many days (null to disable)")
private Integer autoMarkAsReadAfterDays;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
sub.setFilterLegacy(subscription.getFilterLegacy());
sub.setPushNotificationsEnabled(subscription.isPushNotificationsEnabled());
sub.setAutoMarkAsReadAfterDays(subscription.getAutoMarkAsReadAfterDays());
return sub;
}
}
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now()))
? null
: feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
sub.setFilterLegacy(subscription.getFilterLegacy());
sub.setPushNotificationsEnabled(subscription.isPushNotificationsEnabled());
sub.setAutoMarkAsReadAfterDays(subscription.getAutoMarkAsReadAfterDays());
return sub;
}
}

Some files were not shown because too many files have changed in this diff Show More