garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

use our own bidi detector to drop 10Mb from gwt

Browse Source
This commit is contained in:
Athou
2024-09-13 15:54:45 +02:00
parent 19bcc2c0da
commit 31cf4d8bb2
5 changed files with 115 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -16,11 +16,10 @@ import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import com.google.gwt.i18n.client.HasDirection.Direction;
import com.google.gwt.i18n.shared.BidiUtils;
import lombok.extern.slf4j.Slf4j;
@@ -109,8 +108,8 @@ public class FeedUtils {
return false;
}
Direction direction = BidiUtils.get().estimateDirection(text);
return direction == Direction.RTL;
TextDirectionDetector.Direction direction = TextDirectionDetector.detect(text);
return direction == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {

57
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java Normal file
View File

@@ -0,0 +1,57 @@
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
AtomicLong rtl = new AtomicLong();
AtomicLong total = new AtomicLong();
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl.incrementAndGet();
}
}
total.incrementAndGet();
}
if (total.longValue() == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = rtl.doubleValue() / total.doubleValue();
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}