apply formatter

This commit is contained in:
Athou
2013-07-25 09:17:33 +02:00
parent 02f1090fe7
commit 8845c54d0c
82 changed files with 626 additions and 1116 deletions

View File

@@ -27,12 +27,9 @@ public class FaviconFetcher {
private static long MAX_ICON_LENGTH = 20000;
private static int TIMEOUT = 4000;
protected static List<String> ICON_MIMETYPES = Arrays.asList(
"image/x-icon", "image/vnd.microsoft.icon", "image/ico",
"image/icon", "text/ico", "application/ico", "image/x-ms-bmp",
"image/x-bmp", "image/gif", "image/png", "image/jpeg");
private static List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList(
"application/xml", "text/html");
protected static List<String> ICON_MIMETYPES = Arrays.asList("image/x-icon", "image/vnd.microsoft.icon", "image/ico", "image/icon",
"text/ico", "application/ico", "image/x-ms-bmp", "image/x-bmp", "image/gif", "image/png", "image/jpeg");
private static List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
@Inject
HttpGetter getter;
@@ -101,14 +98,12 @@ public class FaviconFetcher {
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length,
MIN_ICON_LENGTH);
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length,
MAX_ICON_LENGTH);
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
@@ -126,8 +121,7 @@ public class FaviconFetcher {
return null;
}
Elements icons = doc
.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);

View File

@@ -30,18 +30,15 @@ public class FeedFetcher {
@Inject
HttpGetter getter;
public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml,
String lastModified, String eTag, Date lastPublishedDate,
String lastContentHash) throws FeedException,
ClientProtocolException, IOException, NotModifiedException {
public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate,
String lastContentHash) throws FeedException, ClientProtocolException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
int timeout = 20000;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag, timeout);
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(
StringUtils.newStringUtf8(result.getContent()), feedUrl);
String extractedUrl = extractFeedUrl(StringUtils.newStringUtf8(result.getContent()), feedUrl);
if (org.apache.commons.lang.StringUtils.isNotBlank(extractedUrl)) {
result = getter.getBinary(extractedUrl, lastModified, eTag, timeout);
feedUrl = extractedUrl;
@@ -54,18 +51,15 @@ public class FeedFetcher {
}
String hash = DigestUtils.sha1Hex(content);
if (lastContentHash != null && hash != null
&& lastContentHash.equals(hash)) {
if (lastContentHash != null && hash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified");
}
fetchedFeed = parser.parse(feedUrl, content);
if (lastPublishedDate != null
&& fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed()
.getLastPublishedDate().getTime()) {
if (lastPublishedDate != null && fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified");
}

View File

@@ -33,12 +33,10 @@ public class FeedParser {
private static Logger log = LoggerFactory.getLogger(FeedParser.class);
private static final String ATOM_10_URI = "http://www.w3.org/2005/Atom";
private static final Namespace ATOM_10_NS = Namespace
.getNamespace(ATOM_10_URI);
private static final Namespace ATOM_10_NS = Namespace.getNamespace(ATOM_10_URI);
private static final Date START = new Date(86400000);
private static final Date END = new Date(
1000l * Integer.MAX_VALUE - 86400000);
private static final Date END = new Date(1000l * Integer.MAX_VALUE - 86400000);
private static final Function<SyndContent, String> CONTENT_TO_STRING = new Function<SyndContent, String>() {
public String apply(SyndContent content) {
@@ -55,11 +53,9 @@ public class FeedParser {
try {
String encoding = FeedUtils.guessEncoding(xml);
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(
xml, encoding));
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url "
+ feedUrl);
throw new FeedException("Input string is null for url " + feedUrl);
}
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed rss = new SyndFeedInput().build(source);
@@ -88,20 +84,16 @@ public class FeedParser {
continue;
}
entry.setGuid(FeedUtils.truncate(guid, 2048));
entry.setUrl(FeedUtils.truncate(
FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink()),
2048));
entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink()), 2048));
entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
FeedEntryContent content = new FeedEntryContent();
content.setContent(getContent(item));
content.setTitle(getTitle(item));
content.setAuthor(item.getAuthor());
SyndEnclosure enclosure = (SyndEnclosure) Iterables.getFirst(
item.getEnclosures(), null);
SyndEnclosure enclosure = (SyndEnclosure) Iterables.getFirst(item.getEnclosures(), null);
if (enclosure != null) {
content.setEnclosureUrl(FeedUtils.truncate(
enclosure.getUrl(), 2048));
content.setEnclosureUrl(FeedUtils.truncate(enclosure.getUrl(), 2048));
content.setEnclosureType(enclosure.getType());
}
entry.setContent(content);
@@ -111,21 +103,17 @@ public class FeedParser {
Date lastEntryDate = null;
Date publishedDate = validateDate(rss.getPublishedDate(), false);
if (!entries.isEmpty()) {
List<Long> sortedTimestamps = FeedUtils
.getSortedTimestamps(entries);
List<Long> sortedTimestamps = FeedUtils.getSortedTimestamps(entries);
Long timestamp = sortedTimestamps.get(0);
lastEntryDate = new Date(timestamp);
publishedDate = getFeedPublishedDate(publishedDate, entries);
}
feed.setLastPublishedDate(validateDate(publishedDate, true));
feed.setAverageEntryInterval(FeedUtils
.averageTimeBetweenEntries(entries));
feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
feed.setLastEntryDate(lastEntryDate);
} catch (Exception e) {
throw new FeedException(String.format(
"Could not parse feed from %s : %s", feedUrl,
e.getMessage()), e);
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
return fetchedFeed;
}
@@ -144,8 +132,7 @@ public class FeedParser {
for (Object object : elements) {
if (object instanceof Element) {
Element element = (Element) object;
if ("link".equals(element.getName())
&& ATOM_10_NS.equals(element.getNamespace())) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
@@ -156,8 +143,7 @@ public class FeedParser {
}
}
private Date getFeedPublishedDate(Date publishedDate,
List<FeedEntry> entries) {
private Date getFeedPublishedDate(Date publishedDate, List<FeedEntry> entries) {
for (FeedEntry entry : entries) {
if (publishedDate == null || entry.getUpdated().getTime() > publishedDate.getTime()) {
@@ -197,12 +183,9 @@ public class FeedParser {
private String getContent(SyndEntry item) {
String content = null;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item
.getDescription().getValue();
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = StringUtils.join(Collections2.transform(
item.getContents(), CONTENT_TO_STRING),
SystemUtils.LINE_SEPARATOR);
content = StringUtils.join(Collections2.transform(item.getContents(), CONTENT_TO_STRING), SystemUtils.LINE_SEPARATOR);
}
return StringUtils.trimToEmpty(content);
}
@@ -224,8 +207,7 @@ public class FeedParser {
private String findHub(SyndFeed feed) {
for (SyndLink l : (List<SyndLink>) feed.getLinks()) {
if ("hub".equalsIgnoreCase(l.getRel())) {
log.debug("found hub {} for feed {}", l.getHref(),
feed.getLink());
log.debug("found hub {} for feed {}", l.getHref(), feed.getLink());
return l.getHref();
}
}
@@ -236,8 +218,7 @@ public class FeedParser {
private String findSelf(SyndFeed feed) {
for (SyndLink l : (List<SyndLink>) feed.getLinks()) {
if ("self".equalsIgnoreCase(l.getRel())) {
log.debug("found self {} for feed {}", l.getHref(),
feed.getLink());
log.debug("found self {} for feed {}", l.getHref(), feed.getLink());
return l.getHref();
}
}

View File

@@ -12,32 +12,28 @@ import org.slf4j.LoggerFactory;
public class FeedRefreshExecutor {
private static Logger log = LoggerFactory
.getLogger(FeedRefreshExecutor.class);
private static Logger log = LoggerFactory.getLogger(FeedRefreshExecutor.class);
private String poolName;
private ThreadPoolExecutor pool;
private LinkedBlockingDeque<Runnable> queue;
public FeedRefreshExecutor(final String poolName, int threads,
int queueCapacity) {
public FeedRefreshExecutor(final String poolName, int threads, int queueCapacity) {
log.info("Creating pool {} with {} threads", poolName, threads);
this.poolName = poolName;
pool = new ThreadPoolExecutor(threads, threads, 0,
TimeUnit.MILLISECONDS,
queue = new LinkedBlockingDeque<Runnable>(queueCapacity) {
private static final long serialVersionUID = 1L;
pool = new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS, queue = new LinkedBlockingDeque<Runnable>(queueCapacity) {
private static final long serialVersionUID = 1L;
@Override
public boolean offer(Runnable r) {
Task task = (Task) r;
if (task.isUrgent()) {
return offerFirst(r);
} else {
return offerLast(r);
}
}
});
@Override
public boolean offer(Runnable r) {
Task task = (Task) r;
if (task.isUrgent()) {
return offerFirst(r);
} else {
return offerLast(r);
}
}
});
pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
@@ -50,8 +46,7 @@ public class FeedRefreshExecutor {
queue.put(r);
}
} catch (InterruptedException e1) {
log.error(poolName
+ " interrupted while waiting for queue.", e1);
log.error(poolName + " interrupted while waiting for queue.", e1);
}
}
});
@@ -80,13 +75,11 @@ public class FeedRefreshExecutor {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error(
"{} interrupted while waiting for threads to finish.",
poolName);
log.error("{} interrupted while waiting for threads to finish.", poolName);
}
}
}
private static class NamedThreadFactory implements ThreadFactory {
private final ThreadGroup group;
private final AtomicInteger threadNumber = new AtomicInteger(1);
@@ -94,15 +87,12 @@ public class FeedRefreshExecutor {
private NamedThreadFactory(String poolName) {
SecurityManager s = System.getSecurityManager();
group = (s != null) ? s.getThreadGroup() :
Thread.currentThread().getThreadGroup();
group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
namePrefix = poolName + "-thread-";
}
public Thread newThread(Runnable r) {
Thread t = new Thread(group, r,
namePrefix + threadNumber.getAndIncrement(),
0);
Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
if (t.isDaemon())
t.setDaemon(false);
if (t.getPriority() != Thread.NORM_PRIORITY)

View File

@@ -52,8 +52,7 @@ public class FeedRefreshTaskGiver {
@PostConstruct
public void init() {
backgroundThreads = applicationSettingsService.get()
.getBackgroundThreads();
backgroundThreads = applicationSettingsService.get().getBackgroundThreads();
executor = Executors.newFixedThreadPool(1);
}
@@ -125,8 +124,7 @@ public class FeedRefreshTaskGiver {
public void add(Feed feed) {
Date threshold = getThreshold();
if (feed.getLastUpdated() == null
|| feed.getLastUpdated().before(threshold)) {
if (feed.getLastUpdated() == null || feed.getLastUpdated().before(threshold)) {
addQueue.add(feed);
}
}

View File

@@ -38,8 +38,7 @@ import com.google.common.util.concurrent.Striped;
@ApplicationScoped
public class FeedRefreshUpdater {
protected static Logger log = LoggerFactory
.getLogger(FeedRefreshUpdater.class);
protected static Logger log = LoggerFactory.getLogger(FeedRefreshUpdater.class);
@Inject
FeedUpdateService feedUpdateService;
@@ -75,8 +74,7 @@ public class FeedRefreshUpdater {
public void init() {
ApplicationSettings settings = applicationSettingsService.get();
int threads = Math.max(settings.getDatabaseUpdateThreads(), 1);
pool = new FeedRefreshExecutor("feed-refresh-updater", threads,
500 * threads);
pool = new FeedRefreshExecutor("feed-refresh-updater", threads, 500 * threads);
locks = Striped.lazyWeakLock(threads * 100000);
}
@@ -113,8 +111,7 @@ public class FeedRefreshUpdater {
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
if (subscriptions == null) {
subscriptions = feedSubscriptionDAO
.findByFeed(feed);
subscriptions = feedSubscriptionDAO.findByFeed(feed);
}
ok &= addEntry(feed, entry, subscriptions);
metricsBean.entryCacheMiss();
@@ -143,8 +140,7 @@ public class FeedRefreshUpdater {
}
}
private boolean addEntry(final Feed feed, final FeedEntry entry,
final List<FeedSubscription> subscriptions) {
private boolean addEntry(final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) {
boolean success = false;
// lock on feed, make sure we are not updating the same feed twice at
@@ -155,8 +151,7 @@ public class FeedRefreshUpdater {
// twice at the same time
String key2 = DigestUtils.sha1Hex(entry.getContent().getContent());
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2))
.iterator();
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
@@ -171,8 +166,7 @@ public class FeedRefreshUpdater {
log.error("lock timeout for " + feed.getUrl() + " - " + key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for " + feed.getUrl()
+ " : " + e.getMessage(), e);
log.error("interrupted while waiting for lock for " + feed.getUrl() + " : " + e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();

View File

@@ -25,8 +25,7 @@ import com.sun.syndication.io.FeedException;
@ApplicationScoped
public class FeedRefreshWorker {
private static Logger log = LoggerFactory
.getLogger(FeedRefreshWorker.class);
private static Logger log = LoggerFactory.getLogger(FeedRefreshWorker.class);
@Inject
FeedRefreshUpdater feedRefreshUpdater;
@@ -52,8 +51,7 @@ public class FeedRefreshWorker {
private void init() {
ApplicationSettings settings = applicationSettingsService.get();
int threads = settings.getBackgroundThreads();
pool = new FeedRefreshExecutor("feed-refresh-worker", threads,
20 * threads);
pool = new FeedRefreshExecutor("feed-refresh-worker", threads, 20 * threads);
}
@PreDestroy
@@ -95,8 +93,7 @@ public class FeedRefreshWorker {
private void update(Feed feed) {
Date now = new Date();
try {
FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false,
feed.getLastModifiedHeader(), feed.getEtagHeader(),
FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is
// thrown
@@ -104,21 +101,17 @@ public class FeedRefreshWorker {
Date disabledUntil = null;
if (applicationSettingsService.get().isHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed
.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
.getAverageEntryInterval());
}
feed.setLastUpdateSuccess(now);
feed.setLink(fetchedFeed.getFeed().getLink());
feed.setLastModifiedHeader(fetchedFeed.getFeed()
.getLastModifiedHeader());
feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader());
feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
feed.setLastContentHash(fetchedFeed.getFeed().getLastContentHash());
feed.setLastPublishedDate(fetchedFeed.getFeed()
.getLastPublishedDate());
feed.setAverageEntryInterval(fetchedFeed.getFeed()
.getAverageEntryInterval());
feed.setLastPublishedDate(fetchedFeed.getFeed().getLastPublishedDate());
feed.setAverageEntryInterval(fetchedFeed.getFeed().getAverageEntryInterval());
feed.setLastEntryDate(fetchedFeed.getFeed().getLastEntryDate());
feed.setErrorCount(0);
@@ -129,14 +122,11 @@ public class FeedRefreshWorker {
feedRefreshUpdater.updateFeed(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(),
e.getMessage());
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
Date disabledUntil = null;
if (applicationSettingsService.get().isHeavyLoad()) {
disabledUntil = FeedUtils
.buildDisabledUntil(feed.getLastEntryDate(),
feed.getAverageEntryInterval());
disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval());
}
feed.setErrorCount(0);
feed.setMessage(null);
@@ -144,8 +134,7 @@ public class FeedRefreshWorker {
taskGiver.giveBack(feed);
} catch (Exception e) {
String message = "Unable to refresh feed " + feed.getUrl() + " : "
+ e.getMessage();
String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
if (e instanceof FeedException) {
log.debug(e.getClass().getName() + " " + message, e);
} else {
@@ -154,8 +143,7 @@ public class FeedRefreshWorker {
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage(message);
feed.setDisabledUntil(FeedUtils.buildDisabledUntil(feed
.getErrorCount()));
feed.setDisabledUntil(FeedUtils.buildDisabledUntil(feed.getErrorCount()));
taskGiver.giveBack(feed);
}

View File

@@ -41,10 +41,8 @@ public class FeedUtils {
protected static Logger log = LoggerFactory.getLogger(FeedUtils.class);
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList(
"height", "width", "border");
private static final char[] DISALLOWED_IFRAME_CSS_RULE_CHARACTERS = new char[] {
'(', ')' };
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final char[] DISALLOWED_IFRAME_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public static String truncate(String string, int length) {
if (string != null) {
@@ -54,8 +52,8 @@ public class FeedUtils {
}
/**
* Detect feed encoding by using the declared encoding in the xml processing
* instruction and by detecting the characters used in the feed
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public static String guessEncoding(byte[] bytes) {
@@ -87,8 +85,7 @@ public class FeedUtils {
}
/**
* Normalize the url. The resulting url is not meant to be fetched but
* rather used as a mean to identify a feed and avoid duplicates
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
@@ -113,13 +110,11 @@ public class FeedUtils {
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com",
"feeds.feedburner.com");
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com",
"feeds.feedburner.com");
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
@@ -146,17 +141,13 @@ public class FeedUtils {
return encoding;
}
public static String handleContent(String content, String baseUri,
boolean keepTextOnly) {
public static String handleContent(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Whitelist whitelist = new Whitelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite",
"code", "col", "colgroup", "dd", "div", "dl", "dt", "em",
"h1", "h2", "h3", "h4", "h5", "h6", "i", "iframe", "img",
"li", "ol", "p", "pre", "q", "small", "strike", "strong",
"sub", "sup", "table", "tbody", "td", "tfoot", "th",
"thead", "tr", "u", "ul");
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em",
"h1", "h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong",
"sub", "sup", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
@@ -167,22 +158,16 @@ public class FeedUtils {
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width",
"allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src",
"title", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor",
"summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr",
"axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr",
"axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https",
"mailto");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
@@ -199,8 +184,7 @@ public class FeedUtils {
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(
EscapeMode.base).prettyPrint(false));
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
@@ -215,9 +199,7 @@ public class FeedUtils {
List<String> rules = Lists.newArrayList();
CSSOMParser parser = new CSSOMParser();
try {
CSSStyleDeclaration decl = parser
.parseStyleDeclaration(new InputSource(new StringReader(
orig)));
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
@@ -226,11 +208,8 @@ public class FeedUtils {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property)
&& StringUtils.containsNone(value,
DISALLOWED_IFRAME_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property)
+ ";");
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, DISALLOWED_IFRAME_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
} catch (IOException e) {
@@ -278,8 +257,7 @@ public class FeedUtils {
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c)
&& !Character.isLowSurrogate(c)) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
@@ -306,8 +284,7 @@ public class FeedUtils {
/**
* When the feed was refreshed successfully
*/
public static Date buildDisabledUntil(Date publishedDate,
Long averageEntryInterval) {
public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval) {
Date now = new Date();
if (publishedDate == null) {
@@ -325,8 +302,7 @@ public class FeedUtils {
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next
int factor = 2;
return new Date(Math.min(DateUtils.addHours(now, 6).getTime(),
now.getTime() + averageEntryInterval / factor));
return new Date(Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor));
} else {
// unknown case, recheck in 24 hours
return DateUtils.addHours(now, 24);
@@ -378,14 +354,11 @@ public class FeedUtils {
return baseUrl + url;
}
public static String getFaviconUrl(FeedSubscription subscription,
String publicUrl) {
return removeTrailingSlash(publicUrl) + "/rest/feed/favicon/"
+ subscription.getId();
public static String getFaviconUrl(FeedSubscription subscription, String publicUrl) {
return removeTrailingSlash(publicUrl) + "/rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content, String publicUrl,
boolean proxyImages) {
public static String proxyImages(String content, String publicUrl, boolean proxyImages) {
if (!proxyImages) {
return content;
}
@@ -398,8 +371,7 @@ public class FeedUtils {
for (Element element : elements) {
String href = element.attr("src");
if (href != null) {
String proxy = removeTrailingSlash(publicUrl)
+ "/rest/server/proxy?u=" + imageProxyEncoder(href);
String proxy = removeTrailingSlash(publicUrl) + "/rest/server/proxy?u=" + imageProxyEncoder(href);
element.attr("src", proxy);
}
}

View File

@@ -28,13 +28,11 @@ public class OPMLExporter {
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.1");
opml.setTitle(String.format("%s subscriptions in CommaFeed",
user.getName()));
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
List<FeedSubscription> subscriptions = feedSubscriptionDAO
.findAll(user);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
for (FeedCategory cat : categories) {
opml.getOutlines().add(buildCategoryOutline(cat, subscriptions));
@@ -50,20 +48,17 @@ public class OPMLExporter {
}
@SuppressWarnings("unchecked")
private Outline buildCategoryOutline(FeedCategory cat,
List<FeedSubscription> subscriptions) {
private Outline buildCategoryOutline(FeedCategory cat, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : cat.getChildren()) {
outline.getChildren().add(
buildCategoryOutline(child, subscriptions));
outline.getChildren().add(buildCategoryOutline(child, subscriptions));
}
for (FeedSubscription sub : subscriptions) {
if (sub.getCategory() != null
&& sub.getCategory().getId().equals(cat.getId())) {
if (sub.getCategory() != null && sub.getCategory().getId().equals(cat.getId())) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
}
@@ -76,11 +71,9 @@ public class OPMLExporter {
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(
new Attribute("xmlUrl", sub.getFeed().getUrl()));
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(
new Attribute("htmlUrl", sub.getFeed().getLink()));
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}

View File

@@ -63,13 +63,12 @@ public class OPMLImporter {
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name,
parent);
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
@@ -91,13 +90,11 @@ public class OPMLImporter {
}
// make sure we continue with the import process even a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(),
name, parent);
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent);
} catch (FeedSubscriptionException e) {
throw e;
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(),
e.getMessage());
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
cache.invalidateUserData(user);