Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-24 04:57:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

only compute rtl once by storing it in the database on fetch

Browse Source
This commit is contained in:
Athou
2024-09-13 22:22:41 +02:00
parent 9a89b39b62
commit f4a43e9950
6 changed files with 42 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -17,7 +17,6 @@ import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
@@ -92,24 +91,18 @@ public class FeedUtils {
return normalized;
}
public static boolean isRTL(FeedEntry entry) {
String text = entry.getContent().getContent();
if (StringUtils.isBlank(text)) {
text = entry.getContent().getTitle();
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
text = Jsoup.parse(text).text();
if (StringUtils.isBlank(text)) {
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
TextDirectionDetector.Direction direction = TextDirectionDetector.detect(text);
return direction == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {

13
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -1,7 +1,6 @@
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
@@ -22,8 +21,8 @@ public class TextDirectionDetector {
return Direction.LEFT_TO_RIGHT;
}
AtomicLong rtl = new AtomicLong();
AtomicLong total = new AtomicLong();
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
@@ -39,18 +38,18 @@ public class TextDirectionDetector {
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl.incrementAndGet();
rtl++;
}
}
total.incrementAndGet();
total++;
}
if (total.longValue() == 0) {
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = rtl.doubleValue() / total.doubleValue();
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}