garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

simplify trie building

Browse Source
This commit is contained in:
Athou
2024-01-07 15:28:48 +01:00
parent da4143fa13
commit a841c80261
1 changed files with 2 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedCleaner.java
View File

@@ -4,7 +4,6 @@ import java.util.Collection;
import org.ahocorasick.trie.Emit; import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie; import org.ahocorasick.trie.Trie;
import org.ahocorasick.trie.Trie.TrieBuilder;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import jakarta.inject.Singleton; import jakarta.inject.Singleton;
@@ -39,19 +38,12 @@ class FeedCleaner {
return sb.toString(); return sb.toString();
} }
// https://stackoverflow.com/a/40836618/1885506
public String replaceHtmlEntitiesWithNumericEntities(String source) { public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized. // Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1); StringBuilder sb = new StringBuilder(source.length() << 1);
TrieBuilder builder = Trie.builder(); Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
builder.ignoreOverlaps();
for (String key : HtmlEntities.HTML_ENTITIES) {
builder.addKeyword(key);
}
Trie trie = builder.build();
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0; int prevIndex = 0;
for (Emit emit : emits) { for (Emit emit : emits) {