From 88fc627dfd347f142a2a87aa984094a4eece5dcf Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Tue, 16 Feb 2021 17:52:50 -0500 Subject: [PATCH 1/9] add back select newly added group --- .../youtube/commentsuite/fxml/ManageGroups.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java index c74be8e..0eddb0c 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java @@ -36,9 +36,6 @@ public class ManageGroups implements Initializable { private static final Logger logger = LogManager.getLogger(); private static final Cache managerCache = CacheBuilder.newBuilder().build(); - private static ManageGroups instance; - - private CommentDatabase database; @FXML private OverlayModal overlayModal; @@ -55,12 +52,8 @@ public class ManageGroups implements Initializable { public void initialize(URL location, ResourceBundle resources) { logger.debug("Initialize ManageGroups"); - instance = this; - CommentSuite.getEventBus().register(this); - database = CommentSuite.getDatabase(); - /* * Logic for main pane. */ @@ -134,6 +127,10 @@ public void groupDeleteEvent(final GroupDeleteEvent deleteEvent) { public void groupAddEvent(final GroupAddEvent addEvent) { logger.debug("Group Add Event"); runLater(this::rebuildGroupSelect); + addEvent.getGroups() + .stream() + .findFirst() + .ifPresent(group -> runLater(() -> comboGroupSelect.setValue(group))); } @Subscribe @@ -144,7 +141,7 @@ public void groupRenameEvent(final GroupRenameEvent renameEvent) { private void rebuildGroupSelect() { final Group selectedGroup = comboGroupSelect.getValue(); - final ObservableList groups = FXCollections.observableArrayList(database.groups().getAllGroups()); + final ObservableList groups = FXCollections.observableArrayList(CommentSuite.getDatabase().groups().getAllGroups()); comboGroupSelect.setItems(FXCollections.emptyObservableList()); comboGroupSelect.setItems(groups); From 3ca02053a19520c42b6ab7f22061d9ccf0d61804 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Sat, 20 Feb 2021 10:52:05 -0500 Subject: [PATCH 2/9] comments and channels: skip missing channel info, sql replace --- .../commentsuite/db/ChannelsTable.java | 4 +- .../commentsuite/db/CommentsTable.java | 4 +- .../youtube/commentsuite/db/SQLLoader.java | 4 +- .../commentsuite/db/YouTubeObject.java | 6 +- .../commentsuite/fxml/SearchYouTube.java | 64 ++++++++----------- .../refresh/CommentThreadProducer.java | 6 +- ...ls.sql => dml_insert_replace_channels.sql} | 2 +- ...ts.sql => dml_insert_replace_comments.sql} | 2 +- 8 files changed, 45 insertions(+), 47 deletions(-) rename src/main/resources/io/mattw/youtube/commentsuite/db/sql/{dml_insert_ignore_channels.sql => dml_insert_replace_channels.sql} (72%) rename src/main/resources/io/mattw/youtube/commentsuite/db/sql/{dml_insert_ignore_comments.sql => dml_insert_replace_comments.sql} (81%) diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/ChannelsTable.java b/src/main/java/io/mattw/youtube/commentsuite/db/ChannelsTable.java index 06bb174..f66de24 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/ChannelsTable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/ChannelsTable.java @@ -11,7 +11,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import static io.mattw.youtube.commentsuite.db.SQLLoader.INSERT_IGNORE_CHANNELS; +import static io.mattw.youtube.commentsuite.db.SQLLoader.INSERT_REPLACE_CHANNELS; public class ChannelsTable extends TableHelper { @@ -95,7 +95,7 @@ public boolean exists(String id) throws SQLException { @Override public void insertAll(List objects) throws SQLException { - try (PreparedStatement ps = preparedStatement(INSERT_IGNORE_CHANNELS.toString())) { + try (PreparedStatement ps = preparedStatement(INSERT_REPLACE_CHANNELS.toString())) { for (YouTubeChannel c : objects) { ps.setString(1, c.getId()); ps.setString(2, c.getTitle()); diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/CommentsTable.java b/src/main/java/io/mattw/youtube/commentsuite/db/CommentsTable.java index bf8af60..bcc8c03 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/CommentsTable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/CommentsTable.java @@ -10,7 +10,7 @@ import java.util.ArrayList; import java.util.List; -import static io.mattw.youtube.commentsuite.db.SQLLoader.INSERT_IGNORE_COMMENTS; +import static io.mattw.youtube.commentsuite.db.SQLLoader.INSERT_REPLACE_COMMENTS; import static org.apache.commons.lang3.StringUtils.isBlank; public class CommentsTable extends TableHelper { @@ -83,7 +83,7 @@ public boolean exists(String id) throws SQLException { @Override public void insertAll(List objects) throws SQLException { - try (PreparedStatement ps = preparedStatement(INSERT_IGNORE_COMMENTS.toString())) { + try (PreparedStatement ps = preparedStatement(INSERT_REPLACE_COMMENTS.toString())) { for (YouTubeComment ct : objects) { ps.setString(1, ct.getId()); ps.setString(2, ct.getChannelId()); diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java b/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java index 1489d97..1cefbfa 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java @@ -46,8 +46,8 @@ public enum SQLLoader { GET_VIDEO_WEEK_HISTOGRAM("dql_get_video_week_histogram.sql"), GROUP_CREATE("dml_create_group.sql"), GROUP_RENAME("dml_rename_group.sql"), - INSERT_IGNORE_CHANNELS("dml_insert_ignore_channels.sql"), - INSERT_IGNORE_COMMENTS("dml_insert_ignore_comments.sql"), + INSERT_REPLACE_CHANNELS("dml_insert_replace_channels.sql"), + INSERT_REPLACE_COMMENTS("dml_insert_replace_comments.sql"), INSERT_IGNORE_GITEM_VIDEO("dml_insert_ignore_gitem_video.sql"), INSERT_IGNORE_MODERATED_COMMENTS("dml_insert_ignore_moderated_comments.sql"), INSERT_REPLACE_VIDEOS("dml_insert_replace_videos.sql"), diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java index 6347ef9..e227d33 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java @@ -7,6 +7,7 @@ import java.io.Serializable; import java.util.Objects; +import java.util.Optional; import java.util.stream.Stream; /** @@ -156,7 +157,10 @@ public static String getChannelIdFromObject(final Object authorChannelId) { return value.get("value"); } - return authorChannelId.toString(); + + return Optional.ofNullable(authorChannelId) + .map(Object::toString) + .orElse(null); } /** diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java index 5e6b120..d4a13a2 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java @@ -35,9 +35,6 @@ public class SearchYouTube implements Initializable { private static final Logger logger = LogManager.getLogger(); private static final String TOKEN_FOO = "TOKEN_FOO"; - - private Location location; - private YouTube youTube; private final ClipboardUtil clipboardUtil = new ClipboardUtil(); private final BrowserUtil browserUtil = new BrowserUtil(); @@ -80,15 +77,12 @@ public class SearchYouTube implements Initializable { public void initialize(URL location, ResourceBundle resources) { logger.debug("Initialize SearchYouTube"); - this.youTube = CommentSuite.getYouTube(); - this.location = CommentSuite.getLocation(); - - MultipleSelectionModel selectionModel = resultsList.getSelectionModel(); + final MultipleSelectionModel selectionModel = resultsList.getSelectionModel(); searchIcon.setImage(ImageLoader.SEARCH.getImage()); geoIcon.setImage(ImageLoader.LOCATION.getImage()); - BooleanBinding isLocation = searchType.valueProperty().isEqualTo("Location"); + final BooleanBinding isLocation = searchType.valueProperty().isEqualTo("Location"); locationBox.managedProperty().bind(isLocation); locationBox.visibleProperty().bind(isLocation); searchRadius.managedProperty().bind(isLocation); @@ -107,13 +101,14 @@ public void initialize(URL location, ResourceBundle resources) { resultsList.getSelectionModel().setSelectionMode(SelectionMode.MULTIPLE); menuCopyId.setOnAction(ae -> { - List list = selectionModel.getSelectedItems(); - List ids = list.stream().map(SearchYouTubeListItem::getObjectId).collect(Collectors.toList()); + final List ids = selectionModel.getSelectedItems().stream() + .map(SearchYouTubeListItem::getObjectId) + .collect(Collectors.toList()); + clipboardUtil.setClipboard(ids); }); menuOpenBrowser.setOnAction(ae -> { - List list = selectionModel.getSelectedItems(); - for (SearchYouTubeListItem view : list) { + for (final SearchYouTubeListItem view : selectionModel.getSelectedItems()) { browserUtil.open(view.getYoutubeURL()); } }); @@ -121,23 +116,20 @@ public void initialize(URL location, ResourceBundle resources) { menuDeselectAll.setOnAction(ae -> selectionModel.clearSelection()); btnAddToGroup.disableProperty().bind(selectionModel.selectedIndexProperty().isEqualTo(-1)); - selectionModel.getSelectedItems().addListener((ListChangeListener) (c -> { - int items = selectionModel.getSelectedItems().size(); - runLater(() -> btnAddToGroup.setText(String.format("Add to Group (%s)", items))); - })); - resultsList.itemsProperty().addListener((o, ov, nv) -> runLater(() -> { - int selectedCount = selectionModel.getSelectedItems().size(); - btnAddToGroup.setText(String.format("Add to Group (%s)", selectedCount)); - })); + selectionModel.getSelectedItems().addListener((ListChangeListener) (c -> + runLater(() -> btnAddToGroup.setText(String.format("Add to Group (%s)", selectionModel.getSelectedItems().size()))) + )); + resultsList.itemsProperty().addListener((o, ov, nv) -> + runLater(() -> btnAddToGroup.setText(String.format("Add to Group (%s)", selectionModel.getSelectedItems().size())) + )); btnClear.setOnAction(ae -> runLater(() -> resultsList.getItems().clear())); geolocate.setOnAction(ae -> new Thread(() -> { geolocate.setDisable(true); try { - IpApiProvider.Location myLocation = this.location.getMyLocation(); - - String coordinates = myLocation.lat + "," + myLocation.lon; + final IpApiProvider.Location myLocation = CommentSuite.getLocation().getMyLocation(); + final String coordinates = myLocation.lat + "," + myLocation.lon; runLater(() -> searchLocation.setText(coordinates)); } catch (IOException e) { @@ -151,17 +143,17 @@ public void initialize(URL location, ResourceBundle resources) { form.setOnKeyPressed(ke -> { if (ke.getCode() == KeyCode.ENTER) { - runLater(() -> - submit.fire() - ); + runLater(submit::fire); } }); submit.setOnAction(ae -> { total = 0; number = 0; pageToken = TOKEN_FOO; - resultsList.getItems().clear(); - runLater(() -> searchInfo.setText(String.format("Showing %s out of %s", resultsList.getItems().size(), total))); + runLater(() -> { + resultsList.getItems().clear(); + searchInfo.setText(String.format("Showing %s out of %s", resultsList.getItems().size(), total)); + }); logger.debug("Submit New Search [pageToken={},type={},text={},locText={},locRadius={},order={},result={}]", pageToken, searchType.getValue(), searchText.getText(), searchLocation.getText(), searchRadius.getValue(), searchOrder.getValue(), resultType.getValue()); @@ -179,7 +171,7 @@ public void initialize(URL location, ResourceBundle resources) { ).start() ); - SYAddToGroupModal syAddToGroupModal = new SYAddToGroupModal(resultsList); + final SYAddToGroupModal syAddToGroupModal = new SYAddToGroupModal(resultsList); addToGroupModal.setContent(syAddToGroupModal); syAddToGroupModal.getBtnClose().setOnAction(ae -> { addToGroupModal.setVisible(false); @@ -199,8 +191,8 @@ public void initialize(URL location, ResourceBundle resources) { public void search(String pageToken, String type, String text, String locText, String locRadius, String order, int resultType) { runLater(() -> searching.setValue(true)); try { - String encodedText = URLEncoder.encode(text, "UTF-8"); - String searchType = types[resultType]; + final String encodedText = URLEncoder.encode(text, "UTF-8"); + final String searchType = types[resultType]; if (pageToken.equals(TOKEN_FOO)) { pageToken = ""; @@ -212,7 +204,7 @@ public void search(String pageToken, String type, String text, String locText, S order = "viewCount"; } - searchList = youTube.search().list("snippet") + searchList = CommentSuite.getYouTube().search().list("snippet") .setKey(CommentSuite.getYouTubeApiKey()) .setMaxResults(50L) .setPageToken(pageToken) @@ -220,7 +212,7 @@ public void search(String pageToken, String type, String text, String locText, S .setType(searchType) .setOrder(order.toLowerCase()); - SearchListResponse sl; + final SearchListResponse sl; if (type.equals("Normal")) { logger.debug("Normal Search [key={},part=snippet,text={},type={},order={},token={}]", CommentSuite.getYouTubeApiKey(), encodedText, searchType, order.toLowerCase(), pageToken); @@ -241,18 +233,18 @@ public void search(String pageToken, String type, String text, String locText, S this.total = sl.getPageInfo().getTotalResults(); logger.debug("Search [videos={}]", sl.getItems().size()); - for (SearchResult item : sl.getItems()) { + for (final SearchResult item : sl.getItems()) { logger.debug("Video [id={},author={},title={}]", item.getId(), item.getSnippet().getChannelTitle(), item.getSnippet().getTitle()); - SearchYouTubeListItem view = new SearchYouTubeListItem(item, number++); + final SearchYouTubeListItem view = new SearchYouTubeListItem(item, number++); runLater(() -> { resultsList.getItems().add(view); searchInfo.setText(String.format("Showing %s out of %s", resultsList.getItems().size(), total)); }); } - } catch (IOException e) { + } catch (final IOException e) { logger.error(e); e.printStackTrace(); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java index e3b2512..07cfd26 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java @@ -154,13 +154,14 @@ private void produce() { final List comments = items.stream() .map(YouTubeComment::new) - .filter(comment -> StringUtils.isNotEmpty(comment.getChannelId())) + .filter(comment -> StringUtils.isNotBlank(comment.getChannelId())) .collect(Collectors.toList()); sendCollection(comments, YouTubeComment.class); final List channels = items.stream() .filter(distinctByKey(CommentThread::getId)) .map(YouTubeChannel::new) + .filter(channel -> StringUtils.isNotBlank(channel.getId())) .collect(Collectors.toList()); sendCollection(channels, YouTubeChannel.class); @@ -185,7 +186,7 @@ private void produce() { final List replies = threadReplies.stream() .map(comment -> new YouTubeComment(comment, video.getId())) - .filter(comment -> StringUtils.isNotEmpty(comment.getChannelId())) + .filter(comment -> StringUtils.isNotBlank(comment.getChannelId())) .collect(Collectors.toList()); sendCollection(replies, YouTubeComment.class); @@ -193,6 +194,7 @@ private void produce() { final List channels2 = threadReplies.stream() .filter(distinctByKey(Comment::getId)) .map(YouTubeChannel::new) + .filter(channel -> StringUtils.isNotBlank(channel.getId())) .collect(Collectors.toList()); sendCollection(channels2, YouTubeChannel.class); } diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_channels.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_channels.sql similarity index 72% rename from src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_channels.sql rename to src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_channels.sql index d9b0707..96423d8 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_channels.sql +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_channels.sql @@ -1,3 +1,3 @@ -INSERT OR IGNORE INTO channels ( +INSERT OR REPLACE INTO channels ( channel_id, channel_name, channel_profile_url, download_profile ) VALUES (?, ?, ?, ?); \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_comments.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_comments.sql similarity index 81% rename from src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_comments.sql rename to src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_comments.sql index 1dc96da..a2168bf 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_comments.sql +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_comments.sql @@ -1,3 +1,3 @@ -INSERT OR IGNORE INTO comments ( +INSERT OR REPLACE INTO comments ( comment_id, channel_id, video_id, comment_date, comment_text, comment_likes, reply_count, is_reply, parent_id ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) \ No newline at end of file From 0e8783923299a3f99a8a54d03c54099439bbd2ca Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Sat, 20 Feb 2021 17:39:48 -0500 Subject: [PATCH 3/9] moderated comments sql replace --- .../youtube/commentsuite/db/GroupItem.java | 45 ++++++++++++++++--- .../db/ModeratedCommentsTable.java | 4 +- .../youtube/commentsuite/db/SQLLoader.java | 24 +++++----- .../commentsuite/db/YouTubeObject.java | 4 +- .../commentsuite/refresh/GroupRefresh.java | 2 +- ...dml_insert_replace_moderated_comments.sql} | 2 +- 6 files changed, 56 insertions(+), 25 deletions(-) rename src/main/resources/io/mattw/youtube/commentsuite/db/sql/{dml_insert_ignore_moderated_comments.sql => dml_insert_replace_moderated_comments.sql} (79%) diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java index 7aa8bd2..5dbde14 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java @@ -10,6 +10,8 @@ import java.io.IOException; import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -110,17 +112,46 @@ public GroupItem(final String link, final boolean fastAdd) throws IOException { ofLink(link); } + enum LinkType { + VIDEO, + PLAYLIST, + CHANNEL_USER, + CHANNEL_ID, + CHANNEL_CUSTOM + } + private void ofLink(final String fullLink) throws IOException { logger.debug("Matching link to type [fullLink={}]", fullLink); youtube = CommentSuite.getYouTube(); database = CommentSuite.getDatabase(); - final Pattern video1 = Pattern.compile("(?:http[s]?://youtu.be/)([\\w_\\-]+)"); - final Pattern video2 = Pattern.compile("(?:http[s]?://www.youtube.com/watch\\?v=)([\\w_\\-]+)"); - final Pattern playlist = Pattern.compile("(?:http[s]?://www.youtube.com/playlist\\?list=)([\\w_\\-]+)"); - final Pattern channel1 = Pattern.compile("(?:http[s]?://www.youtube.com/channel/)([\\w_\\-]+)"); - final Pattern channel2 = Pattern.compile("(?:http[s]?://www.youtube.com/user/)([\\w_\\-]+)"); + final Pattern video1 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/watch\\?v=([\\w_-]+)(?:&.*)?"); + final Pattern video2 = Pattern.compile("(?:http[s]?://)?youtu.be/([\\w_-]+)(?:\\?.*)?"); + final Pattern playlist = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/playlist\\?list=([\\w_-]+)(?:&.*)?"); + final Pattern channelUser = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/user/([\\w_-]+)(?:\\?.*)?"); + final Pattern channelId = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/channel/([\\w_-]+)(?:\\?.*)?"); + final Pattern channelCustom1 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/c/([\\w_-]+)(?:\\?.*)?"); + final Pattern channelCustom2 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/([\\w_-]+)(?:\\?.*)?"); + + final Map patterns = new HashMap<>(); + patterns.put(video1, LinkType.VIDEO); + patterns.put(video2, LinkType.VIDEO); + patterns.put(playlist, LinkType.PLAYLIST); + patterns.put(channelUser, LinkType.CHANNEL_USER); + patterns.put(channelId, LinkType.CHANNEL_ID); + patterns.put(channelCustom1, LinkType.CHANNEL_CUSTOM); + patterns.put(channelCustom2, LinkType.CHANNEL_CUSTOM); + + String linkId = null; + LinkType linkType = null; + for (final Pattern pattern : patterns.keySet()) { + final Matcher matcher = pattern.matcher(fullLink); + if (matcher.matches()) { + linkId = matcher.group(1); + linkType = patterns.get(pattern); + } + } Matcher m; YType type = YType.UNKNOWN; @@ -135,10 +166,10 @@ private void ofLink(final String fullLink) throws IOException { } else if ((m = playlist.matcher(fullLink)).matches()) { result = m.group(1); type = YType.PLAYLIST; - } else if ((m = channel1.matcher(fullLink)).matches()) { + } else if ((m = channelUser.matcher(fullLink)).matches()) { result = m.group(1); type = YType.CHANNEL; - } else if ((m = channel2.matcher(fullLink)).matches()) { + } else if ((m = channelId.matcher(fullLink)).matches()) { result = m.group(1); type = YType.CHANNEL; channelUsername = true; diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/ModeratedCommentsTable.java b/src/main/java/io/mattw/youtube/commentsuite/db/ModeratedCommentsTable.java index 93a0c19..a141611 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/ModeratedCommentsTable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/ModeratedCommentsTable.java @@ -7,7 +7,7 @@ import java.util.ArrayList; import java.util.List; -import static io.mattw.youtube.commentsuite.db.SQLLoader.INSERT_IGNORE_MODERATED_COMMENTS; +import static io.mattw.youtube.commentsuite.db.SQLLoader.INSERT_REPLACE_MODERATED_COMMENTS; public class ModeratedCommentsTable extends TableHelper { @@ -69,7 +69,7 @@ public boolean exists(String id) throws SQLException { @Override public void insertAll(List objects) throws SQLException { - try (PreparedStatement ps = preparedStatement(INSERT_IGNORE_MODERATED_COMMENTS.toString())) { + try (PreparedStatement ps = preparedStatement(INSERT_REPLACE_MODERATED_COMMENTS.toString())) { for (YouTubeComment ct : objects) { ps.setString(1, ct.getId()); ps.setString(2, ct.getChannelId()); diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java b/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java index 1cefbfa..17c4f9b 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java @@ -6,6 +6,8 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; /** * Loads various resource SQL scripts for definition, query, and manipulation. @@ -49,7 +51,7 @@ public enum SQLLoader { INSERT_REPLACE_CHANNELS("dml_insert_replace_channels.sql"), INSERT_REPLACE_COMMENTS("dml_insert_replace_comments.sql"), INSERT_IGNORE_GITEM_VIDEO("dml_insert_ignore_gitem_video.sql"), - INSERT_IGNORE_MODERATED_COMMENTS("dml_insert_ignore_moderated_comments.sql"), + INSERT_REPLACE_MODERATED_COMMENTS("dml_insert_replace_moderated_comments.sql"), INSERT_REPLACE_VIDEOS("dml_insert_replace_videos.sql"), RESET_DB("ddl_reset_db.sql"), UPDATE_GITEM("dml_update_gitem.sql"), @@ -58,25 +60,25 @@ public enum SQLLoader { private final Logger logger = LogManager.getLogger(); - private String basePath = "/io/mattw/youtube/commentsuite/db/sql/"; - private String fileName; - private String fileData = ""; + private final String fileName; + private final String fileData; - SQLLoader(String fileName) { + SQLLoader(final String fileName) { this.fileName = fileName; + + final List lines = new ArrayList<>(); try { String line; - StringBuilder sb = new StringBuilder(); - try (InputStreamReader isr = new InputStreamReader(getClass().getResource(basePath + fileName).openStream()); - BufferedReader br = new BufferedReader(isr)) { + try (final InputStreamReader isr = new InputStreamReader(getClass().getResourceAsStream("/io/mattw/youtube/commentsuite/db/sql/" + fileName)); + final BufferedReader br = new BufferedReader(isr)) { while ((line = br.readLine()) != null) { - sb.append(line); - sb.append(' '); + lines.add(line); } - this.fileData = sb.toString(); } } catch (IOException e) { logger.error("Failed to load resource sql file [fileKey={}]", getFileName()); + } finally { + this.fileData = String.join(" ", lines); } } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java index e227d33..051c756 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java @@ -153,9 +153,7 @@ public boolean equals(Object o) { */ public static String getChannelIdFromObject(final Object authorChannelId) { if (authorChannelId instanceof ArrayMap) { - ArrayMap value = (ArrayMap) authorChannelId; - - return value.get("value"); + return ((ArrayMap) authorChannelId).get("value"); } return Optional.ofNullable(authorChannelId) diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java index a316434..48ae372 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java @@ -187,7 +187,7 @@ public void run() { channelProducer.getDuplicateSkipped()); } - private void await(ConsumerMultiProducer consumer, String message) throws InterruptedException { + private void await(final ConsumerMultiProducer consumer, final String message) throws InterruptedException { if (consumer.getExecutorGroup().isStillWorking()) { consumer.getExecutorGroup().await(); consumer.onCompletion(); diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_moderated_comments.sql similarity index 79% rename from src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql rename to src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_moderated_comments.sql index 9048aec..7b44648 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_replace_moderated_comments.sql @@ -1,3 +1,3 @@ -INSERT OR IGNORE INTO comments_moderated ( +INSERT OR REPLACE INTO comments_moderated ( comment_id, channel_id, video_id, comment_date, comment_text, comment_likes, reply_count, is_reply, parent_id, moderation_status ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ No newline at end of file From 7c8f5d046f6e6d96ee168f902cbcdf1660332915 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Mon, 22 Feb 2021 22:01:31 -0500 Subject: [PATCH 4/9] group item resolver; faster bulk add resolving; better link resolving --- .../youtube/commentsuite/db/CommentQuery.java | 2 +- .../youtube/commentsuite/db/GroupItem.java | 308 +++------------ .../db/GroupItemLinkResolver.java | 365 ++++++++++++++++++ .../commentsuite/db/GroupItemResolver.java | 176 +++++++++ .../db/{YType.java => GroupItemType.java} | 4 +- .../commentsuite/db/GroupItemsTable.java | 23 +- .../youtube/commentsuite/db/LinkType.java | 19 + .../youtube/commentsuite/db/Linkable.java | 17 + .../commentsuite/db/YouTubeChannel.java | 8 +- .../commentsuite/db/YouTubeComment.java | 6 +- .../commentsuite/db/YouTubeObject.java | 33 +- .../youtube/commentsuite/db/YouTubeVideo.java | 4 +- .../commentsuite/fxml/MGMVAddItemModal.java | 89 ++--- .../commentsuite/fxml/MGMVGroupItemView.java | 10 +- .../commentsuite/fxml/ManageGroups.java | 2 - .../commentsuite/fxml/SCVideoSelectModal.java | 2 +- .../commentsuite/fxml/SYAddToGroupModal.java | 23 +- .../commentsuite/fxml/SearchComments.java | 4 +- .../commentsuite/refresh/VideoIdProducer.java | 10 +- .../commentsuite/fxml/MainQuotaModal.fxml | 6 +- 20 files changed, 721 insertions(+), 390 deletions(-) create mode 100644 src/main/java/io/mattw/youtube/commentsuite/db/GroupItemLinkResolver.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/db/GroupItemResolver.java rename src/main/java/io/mattw/youtube/commentsuite/db/{YType.java => GroupItemType.java} (86%) create mode 100644 src/main/java/io/mattw/youtube/commentsuite/db/LinkType.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/db/Linkable.java diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java b/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java index d0dcf53..4302d2f 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java @@ -386,7 +386,7 @@ public void prepForExport() { if (groupItem.isPresent()) { GroupItem item = groupItem.get(); - this.withGroupItem = String.format("%s / %s", item.getId(), item.getTitle()); + this.withGroupItem = String.format("%s / %s", item.getId(), item.getDisplayName()); } else { this.withGroupItem = "All Item(s)"; } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java index 5dbde14..728f28b 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java @@ -1,314 +1,98 @@ package io.mattw.youtube.commentsuite.db; -import com.google.api.services.youtube.YouTube; -import com.google.api.services.youtube.model.*; -import io.mattw.youtube.commentsuite.CommentSuite; -import io.mattw.youtube.commentsuite.ConfigData; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.Objects; +import java.util.stream.Stream; /** * Database entry for searched YouTube entries (Video, Channel, Playlist). * getYouTubeId() from YouTubeObject represents the GroupItem ID. */ -public class GroupItem extends YouTubeObject { - - private static final Logger logger = LogManager.getLogger(); +public class GroupItem implements Linkable { - public static String ALL_ITEMS = "ALL_ITEMS"; + public static final String ALL_ITEMS = "ALL_ITEMS"; + private String id; + private String displayName; + private String thumbUrl; + private GroupItemType type; private String channelTitle; private long published; private long lastChecked; - private YouTube youtube; - private CommentDatabase database; - private boolean fastAdd = false; - - /** - * Used for converting selected search items for inserting into database. - */ - public GroupItem(final SearchResult item) { - super(item.getId(), item.getSnippet().getTitle(), - item.getSnippet().getThumbnails().getMedium().getUrl()); - - this.published = item.getSnippet().getPublishedAt().getValue(); - this.channelTitle = item.getSnippet().getChannelTitle(); - this.lastChecked = 0; - - if (item.getId().getVideoId() != null) setTypeId(YType.VIDEO); - if (item.getId().getChannelId() != null) setTypeId(YType.CHANNEL); - if (item.getId().getPlaylistId() != null) setTypeId(YType.PLAYLIST); - } - - /** - * Used by ManageGroupsManager when user submits link directly. - * - * @param item VideosList.Item - */ - public GroupItem(final Video item) { - this(item.getId(), YType.VIDEO, item.getSnippet().getTitle(), item.getSnippet().getChannelTitle(), - item.getSnippet().getThumbnails().getMedium().getUrl(), item.getSnippet().getPublishedAt().getValue(), 0); - } - - /** - * Used by ManageGroupsManager when user submits link directly. - * - * @param item ChannelsList.Item - */ - public GroupItem(final Channel item) { - this(item.getId(), YType.CHANNEL, item.getSnippet().getTitle(), item.getSnippet().getTitle(), - item.getSnippet().getThumbnails().getMedium().getUrl(), item.getSnippet().getPublishedAt().getValue(), 0); - } - - /** - * Used by ManageGroupsManager when user submits link directly. - * - * @param item PlaylistItemsList.Item - */ - public GroupItem(final Playlist item) { - this(item.getId(), YType.PLAYLIST, item.getSnippet().getTitle(), item.getSnippet().getChannelTitle(), - item.getSnippet().getThumbnails().getMedium().getUrl(), item.getSnippet().getPublishedAt().getValue(), 0); + public String getId() { + return id; } - /** - * Used for "All Items (#)" and "No items" display in the "Comment Search" and "Group Manager" ComboBoxes. - */ - public GroupItem(final String gitemId, final String title) { - super(gitemId, title, null); + public GroupItem setId(String id) { + this.id = id; + return this; } - /** - * Used by the database when querying for group items. - */ - public GroupItem(final String gitemId, final YType typeId, final String title, final String channelTitle, final String thumbUrl, final long published, final long lastChecked) { - super(gitemId, title, thumbUrl); - setTypeId(typeId); - this.channelTitle = channelTitle; - this.published = published; - this.lastChecked = lastChecked; + public String getDisplayName() { + return displayName; } - /** - * Parses video, playlist, and channel links to create a GroupItem. - * - * @param link a video, playlist, or channel link - * @throws IOException there was a problem parsing the link - */ - public GroupItem(final String link) throws IOException { - ofLink(link); + public GroupItem setDisplayName(String displayName) { + this.displayName = displayName; + return this; } - public GroupItem(final String link, final boolean fastAdd) throws IOException { - this.fastAdd = fastAdd; - ofLink(link); + public String getThumbUrl() { + return thumbUrl; } - enum LinkType { - VIDEO, - PLAYLIST, - CHANNEL_USER, - CHANNEL_ID, - CHANNEL_CUSTOM - } - - private void ofLink(final String fullLink) throws IOException { - logger.debug("Matching link to type [fullLink={}]", fullLink); - - youtube = CommentSuite.getYouTube(); - database = CommentSuite.getDatabase(); - - final Pattern video1 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/watch\\?v=([\\w_-]+)(?:&.*)?"); - final Pattern video2 = Pattern.compile("(?:http[s]?://)?youtu.be/([\\w_-]+)(?:\\?.*)?"); - final Pattern playlist = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/playlist\\?list=([\\w_-]+)(?:&.*)?"); - final Pattern channelUser = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/user/([\\w_-]+)(?:\\?.*)?"); - final Pattern channelId = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/channel/([\\w_-]+)(?:\\?.*)?"); - final Pattern channelCustom1 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/c/([\\w_-]+)(?:\\?.*)?"); - final Pattern channelCustom2 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/([\\w_-]+)(?:\\?.*)?"); - - final Map patterns = new HashMap<>(); - patterns.put(video1, LinkType.VIDEO); - patterns.put(video2, LinkType.VIDEO); - patterns.put(playlist, LinkType.PLAYLIST); - patterns.put(channelUser, LinkType.CHANNEL_USER); - patterns.put(channelId, LinkType.CHANNEL_ID); - patterns.put(channelCustom1, LinkType.CHANNEL_CUSTOM); - patterns.put(channelCustom2, LinkType.CHANNEL_CUSTOM); - - String linkId = null; - LinkType linkType = null; - for (final Pattern pattern : patterns.keySet()) { - final Matcher matcher = pattern.matcher(fullLink); - if (matcher.matches()) { - linkId = matcher.group(1); - linkType = patterns.get(pattern); - } - } - - Matcher m; - YType type = YType.UNKNOWN; - boolean channelUsername = false; - String result = ""; - if ((m = video1.matcher(fullLink)).matches()) { - result = m.group(1); - type = YType.VIDEO; - } else if ((m = video2.matcher(fullLink)).matches()) { - result = m.group(1); - type = YType.VIDEO; - } else if ((m = playlist.matcher(fullLink)).matches()) { - result = m.group(1); - type = YType.PLAYLIST; - } else if ((m = channelUser.matcher(fullLink)).matches()) { - result = m.group(1); - type = YType.CHANNEL; - } else if ((m = channelId.matcher(fullLink)).matches()) { - result = m.group(1); - type = YType.CHANNEL; - channelUsername = true; - } - - if (fastAdd) { - if (channelUsername) { - throw new IOException("Channel usernames are not accepted when using fast group add."); - } - - setId(result); - setTitle(result); - setTypeId(type); - setChannelTitle(StringUtils.EMPTY); - setThumbUrl(ConfigData.FAST_GROUP_ADD_THUMB_PLACEHOLDER); - setPublished(0); - setLastChecked(0); - - return; - } - - - final YouTube youtube = CommentSuite.getYouTube(); - if (result.isEmpty()) { - throw new IOException(String.format("Input did not match expected formats [fullLink=%s]", fullLink)); - } else { - if (type == YType.VIDEO) { - final VideoListResponse vl = youtube.videos().list("snippet") - .setKey(CommentSuite.getYouTubeApiKey()) - .setId(result) - .execute(); - - if (!vl.getItems().isEmpty()) { - final Video item = vl.getItems().get(0); - - duplicate(new GroupItem(item)); - - checkForNewChannel(item.getSnippet().getChannelId()); - } - } else if (type == YType.CHANNEL) { - YouTube.Channels.List cl = youtube.channels().list("snippet") - .setKey(CommentSuite.getYouTubeApiKey()); - - if (!channelUsername) { - cl = cl.setId(result); - } else { - cl = cl.setForUsername(result); - } - - final ChannelListResponse clr = cl.execute(); - - if (!clr.getItems().isEmpty()) { - final Channel item = clr.getItems().get(0); - - duplicate(new GroupItem(item)); - - checkForNewChannel(item.getId()); - } - } else { - final PlaylistListResponse pl = youtube.playlists().list("snippet") - .setKey(CommentSuite.getYouTubeApiKey()) - .setId(result) - .execute(); - - if (!pl.getItems().isEmpty()) { - final Playlist item = pl.getItems().get(0); - - duplicate(new GroupItem(item)); - - checkForNewChannel(item.getSnippet().getChannelId()); - } - } - } + public GroupItem setThumbUrl(String thumbUrl) { + this.thumbUrl = thumbUrl; + return this; } - /** - * Makes sure the channel associated with this GroupItem is in the database. - */ - private void checkForNewChannel(final String channelId) { - try { - if (database.channels().notExists(channelId)) { - final ChannelListResponse clr = youtube.channels().list("snippet") - .setKey(CommentSuite.getYouTubeApiKey()) - .setId(channelId) - .execute(); - - final YouTubeChannel channel = new YouTubeChannel(clr.getItems().get(0)); - - database.channels().insert(channel); - } - } catch (IOException e) { - logger.error("Failed to get channel [id={}]", channelId); - } catch (SQLException e) { - logger.error("Failed to insert new channel [id={}]", channelId); - } + public GroupItemType getType() { + return type; } - /** - * Copies fields of submitted GroupItem to this instance's fields. - * - * @param groupItem item to copy fields from - */ - private void duplicate(GroupItem groupItem) { - setTypeId(groupItem.getTypeId()); - setId(groupItem.getId()); - setTitle(groupItem.getTitle()); - setThumbUrl(groupItem.getThumbUrl()); - setYouTubeLink(groupItem.getYouTubeLink()); - - setChannelTitle(groupItem.getChannelTitle()); - setLastChecked(groupItem.getLastChecked()); - setPublished(groupItem.getPublished()); + public GroupItem setType(GroupItemType type) { + this.type = type; + return this; } public String getChannelTitle() { return channelTitle; } - public void setChannelTitle(String channelTitle) { + public GroupItem setChannelTitle(String channelTitle) { this.channelTitle = channelTitle; + return this; } public long getPublished() { return published; } - public void setPublished(long published) { + public GroupItem setPublished(long published) { this.published = published; + return this; } public long getLastChecked() { return lastChecked; } - public void setLastChecked(long timestamp) { - this.lastChecked = timestamp; + public GroupItem setLastChecked(long lastChecked) { + this.lastChecked = lastChecked; + return this; } + @Override public String toString() { - return getTitle(); + return Stream.of(displayName, id) + .filter(Objects::nonNull) + .findFirst() + .orElse(super.toString()); } + + @Override + public String toYouTubeLink() { + return String.format(GROUP_ITEM_FORMATS.get(this.type), this.id); + } + } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemLinkResolver.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemLinkResolver.java new file mode 100644 index 0000000..1d3b035 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemLinkResolver.java @@ -0,0 +1,365 @@ +package io.mattw.youtube.commentsuite.db; + +import com.google.api.services.youtube.YouTube; +import com.google.api.services.youtube.model.*; +import io.mattw.youtube.commentsuite.CommentSuite; +import io.mattw.youtube.commentsuite.util.ExecutorGroup; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +public class GroupItemLinkResolver { + + private static final Logger logger = LogManager.getLogger(); + + private static final int THREADS_PER_TYPE = 3; + private static final GroupItemResolver resolver = new GroupItemResolver(); + private static final Map linkPatterns = new HashMap<>(); + static { + final Pattern video1 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/watch\\?v=([\\w_-]+)(?:&.*)?"); + final Pattern video2 = Pattern.compile("(?:http[s]?://)?youtu.be/([\\w_-]+)(?:\\?.*)?"); + final Pattern playlist = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/playlist\\?list=([\\w_-]+)(?:&.*)?"); + final Pattern channelUser = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/user/([\\w_-]+)(?:\\?.*)?"); + final Pattern channelId = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/channel/([\\w_-]+)(?:\\?.*)?"); + final Pattern channelCustom1 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/c/([\\w_-]+)(?:\\?.*)?"); + final Pattern channelCustom2 = Pattern.compile("(?:http[s]?://)?(?:\\w+\\.)?youtube.com/([\\w_-]+)(?:\\?.*)?"); + + linkPatterns.put(video1, LinkType.VIDEO); + linkPatterns.put(video2, LinkType.VIDEO); + linkPatterns.put(playlist, LinkType.PLAYLIST); + linkPatterns.put(channelUser, LinkType.CHANNEL_USER); + linkPatterns.put(channelId, LinkType.CHANNEL_ID); + linkPatterns.put(channelCustom1, LinkType.CHANNEL_CUSTOM); + linkPatterns.put(channelCustom2, LinkType.CHANNEL_CUSTOM); + } + + private final CommentDatabase database; + private final YouTube youTube; + + public GroupItemLinkResolver() { + this.database = CommentSuite.getDatabase(); + this.youTube = CommentSuite.getYouTube(); + } + + public Optional from(final String link) { + return parseAndAwait(Collections.singleton(link)).stream().findFirst(); + } + + public List from(final Collection links) { + return parseAndAwait(links); + } + + private List parseAndAwait(final Collection links) { + if (links == null || links.isEmpty()) { + return Collections.emptyList(); + } + + final Map matches = new HashMap<>(); + for (final String link : links.stream().distinct().collect(Collectors.toList())) { + if (isBlank(link)) { + continue; + } + + for (final Pattern pattern : linkPatterns.keySet()) { + final Matcher matcher = pattern.matcher(link); + if (matcher.matches()) { + matches.put(matcher.group(1), linkPatterns.get(pattern)); + } + } + } + + if (matches.isEmpty()) { + return Collections.emptyList(); + } + + final List results = new ArrayList<>(); + final ExecutorGroup videoExecutor = parseVideos(matches, results); + final ExecutorGroup playlistsExecutor = parsePlaylists(matches, results); + final ExecutorGroup channelIdsExecutor = parseChannelIds(matches, results); + final ExecutorGroup channelUsersExecutor = parseChannelUsers(matches, results); + final ExecutorGroup channelCustomsExecutor = parseChannelCustoms(matches, results); + + try { + awaitAll(videoExecutor, playlistsExecutor, channelIdsExecutor, channelUsersExecutor, channelCustomsExecutor); + } catch (InterruptedException e) { + logger.error(e); + } + + logger.debug(results); + + return results; + } + + private void awaitAll(final ExecutorGroup... executors) throws InterruptedException { + for (ExecutorGroup executor : executors) { + if (executor.isStillWorking()) { + executor.await(); + } + } + } + + private LinkedBlockingQueue toQueue(final Map matches, final LinkType type) { + return matches.entrySet().stream() + .filter(entry -> entry.getValue() == type) + .map(Map.Entry::getKey) + .collect(Collectors.toCollection(LinkedBlockingQueue::new)); + } + + private ExecutorGroup parseVideos(final Map matches, final List results) { + final LinkedBlockingQueue ids = toQueue(matches, LinkType.VIDEO); + final ExecutorGroup executors = new ExecutorGroup(THREADS_PER_TYPE); + executors.submitAndShutdown(() -> { + final List toCheck = new ArrayList<>(); + while (!ids.isEmpty()) { + final String id = ids.poll(); + if (id == null) { + continue; + } else { + toCheck.add(id); + } + + if (toCheck.size() == 50) { + checkVideoIds(toCheck, results); + toCheck.clear(); + } + } + + if (!toCheck.isEmpty()) { + checkVideoIds(toCheck, results); + toCheck.clear(); + } + }); + return executors; + } + + private void checkVideoIds(final List ids, final List results) { + try { + logger.debug("checkVideoIds({})", ids); + + final VideoListResponse videos = youTube.videos() + .list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) + .setId(String.join(",", ids)) + .setMaxResults(50L) + .execute(); + + Optional.ofNullable(videos) + .map(response -> response.getItems().stream()) + .orElseGet(Stream::empty) + .map(resolver::from) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(results::add); + } catch (IOException e) { + logger.error(e); + } + } + + private ExecutorGroup parsePlaylists(final Map matches, final List results) { + final LinkedBlockingQueue ids = toQueue(matches, LinkType.PLAYLIST); + final ExecutorGroup executors = new ExecutorGroup(THREADS_PER_TYPE); + executors.submitAndShutdown(() -> { + final List toCheck = new ArrayList<>(); + while (!ids.isEmpty()) { + final String id = ids.poll(); + if (id == null) { + continue; + } else { + toCheck.add(id); + } + + if (toCheck.size() == 50) { + checkPlaylistIds(toCheck, results); + toCheck.clear(); + } + } + + if (!toCheck.isEmpty()) { + checkPlaylistIds(toCheck, results); + toCheck.clear(); + } + }); + return executors; + } + + private void checkPlaylistIds(final List ids, final List results) { + try { + logger.debug("checkPlaylistIds({})", ids); + + final PlaylistListResponse playlists = youTube.playlists() + .list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) + .setId(String.join(",", ids)) + .setMaxResults(50L) + .execute(); + + Optional.ofNullable(playlists) + .map(response -> response.getItems().stream()) + .orElseGet(Stream::empty) + .map(resolver::from) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(results::add); + } catch (IOException e) { + logger.error(e); + } + } + + private ExecutorGroup parseChannelIds(final Map matches, final List results) { + final LinkedBlockingQueue ids = toQueue(matches, LinkType.CHANNEL_ID); + final ExecutorGroup executors = new ExecutorGroup(THREADS_PER_TYPE); + executors.submitAndShutdown(() -> { + final List toCheck = new ArrayList<>(); + while (!ids.isEmpty()) { + final String id = ids.poll(); + if (id == null) { + continue; + } else { + toCheck.add(id); + } + + if (toCheck.size() == 50) { + checkChannelIds(toCheck, results); + toCheck.clear(); + } + } + + if (!toCheck.isEmpty()) { + checkChannelIds(toCheck, results); + toCheck.clear(); + } + }); + return executors; + } + + private void checkChannelIds(final List ids, final List results) { + try { + logger.debug("checkChannelIds({})", ids); + + final ChannelListResponse channels = youTube.channels() + .list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) + .setId(String.join(",", ids)) + .setMaxResults(50L) + .execute(); + + checkChannelResponse(channels, results); + } catch (Exception e) { + logger.error(e); + } + } + + private ExecutorGroup parseChannelUsers(final Map matches, final List results) { + final LinkedBlockingQueue ids = toQueue(matches, LinkType.CHANNEL_USER); + final ExecutorGroup executors = new ExecutorGroup(THREADS_PER_TYPE); + executors.submitAndShutdown(() -> { + while (!ids.isEmpty()) { + final String id = ids.poll(); + if (id == null) { + continue; + } + + checkChannelUser(id, results); + } + }); + return executors; + } + + private void checkChannelUser(final String id, final List results) { + try { + logger.debug("checkChannelUser({})", id); + + final ChannelListResponse channels = youTube.channels() + .list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) + .setForUsername(id) + .execute(); + + checkChannelResponse(channels, results); + } catch (Exception e) { + logger.error(e); + } + } + + private ExecutorGroup parseChannelCustoms(final Map matches, final List results) { + final LinkedBlockingQueue ids = toQueue(matches, LinkType.CHANNEL_CUSTOM); + final ExecutorGroup executors = new ExecutorGroup(THREADS_PER_TYPE); + executors.submitAndShutdown(() -> { + while (!ids.isEmpty()) { + final String id = ids.poll(); + if (id == null) { + continue; + } + + checkChannelCustom(id, results); + } + }); + return executors; + } + + /** + * Cannot directly query by customUrl so have to workaround by checking channels returned from + * search.list. However, that extra detail can only be found from channels.list not search.list. + */ + private void checkChannelCustom(final String id, final List results) { + try { + logger.debug("checkChannelCustom({})", id); + + final SearchListResponse search = youTube.search() + .list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) + .setQ(id) + .setType("channel") + .setMaxResults(50L) + .execute(); + + final List channelIds = Optional.ofNullable(search) + .map(res -> res.getItems().stream()) + .orElseGet(Stream::empty) + .map(SearchResult::getSnippet) + .map(SearchResultSnippet::getChannelId) + .distinct() + .collect(Collectors.toList()); + + final ChannelListResponse channels = youTube.channels() + .list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) + .setId(String.join(",", channelIds)) + .setMaxResults(50L) + .execute(); + + final Optional match = Optional.ofNullable(channels) + .map(res -> res.getItems().stream()) + .orElseGet(Stream::empty) + .filter(channel -> StringUtils.equalsIgnoreCase(id, channel.getSnippet().getCustomUrl())) + .findFirst() + .flatMap(resolver::from); + + match.ifPresent(results::add); + } catch (Exception e) { + logger.error(e); + } + } + + private void checkChannelResponse(final ChannelListResponse response, final List results) { + Optional.ofNullable(response) + .map(res -> res.getItems().stream()) + .orElseGet(Stream::empty) + .map(resolver::from) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(results::add); + } + + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemResolver.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemResolver.java new file mode 100644 index 0000000..fef4de9 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemResolver.java @@ -0,0 +1,176 @@ +package io.mattw.youtube.commentsuite.db; + +import com.google.api.client.util.DateTime; +import com.google.api.services.youtube.model.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.*; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.isBlank; + +/** + * Resolve a number of different data types from the YouTube API and links to GroupItem(s) + */ +public class GroupItemResolver { + + private static final Logger logger = LogManager.getLogger(); + + public Optional to( + final String id, final GroupItemType type, final String displayName, + final String thumbUrl, final String channelTitle, final long publishedAt + ) { + if (isBlank(id) || type == null) { + return Optional.empty(); + } + + return Optional.of(new GroupItem() + .setId(id) + .setType(type) + .setDisplayName(displayName) + .setThumbUrl(thumbUrl) + .setChannelTitle(channelTitle) + .setPublished(publishedAt)); + } + + public Optional from(final SearchResult searchResult) { + final String id = Optional.ofNullable(searchResult) + .map(SearchResult::getId) + .map(this::getIdFromResource) + .orElse(null); + final GroupItemType type = Optional.ofNullable(searchResult) + .map(SearchResult::getId) + .map(this::getTypeFromResource) + .orElse(null); + final String displayName = Optional.ofNullable(searchResult) + .map(SearchResult::getSnippet) + .map(SearchResultSnippet::getTitle) + .orElse(null); + final String thumbUrl = Optional.ofNullable(searchResult) + .map(SearchResult::getSnippet) + .map(SearchResultSnippet::getThumbnails) + .map(ThumbnailDetails::getMedium) + .map(Thumbnail::getUrl) + .orElse(null); + final String channelTitle = Optional.ofNullable(searchResult) + .map(SearchResult::getSnippet) + .map(SearchResultSnippet::getChannelTitle) + .orElse(null); + final long published = Optional.ofNullable(searchResult) + .map(SearchResult::getSnippet) + .map(SearchResultSnippet::getPublishedAt) + .map(DateTime::getValue) + .orElse(0L); + + return to(id, type, displayName, thumbUrl, channelTitle, published); + } + + public Optional from(final Video video) { + final String id = Optional.ofNullable(video) + .map(Video::getId) + .orElse(null); + final String displayName = Optional.ofNullable(video) + .map(Video::getSnippet) + .map(VideoSnippet::getTitle) + .orElse(null); + final String thumbUrl = Optional.ofNullable(video) + .map(Video::getSnippet) + .map(VideoSnippet::getThumbnails) + .map(ThumbnailDetails::getMedium) + .map(Thumbnail::getUrl) + .orElse(null); + final String channelTitle = Optional.ofNullable(video) + .map(Video::getSnippet) + .map(VideoSnippet::getChannelTitle) + .orElse(null); + final long published = Optional.ofNullable(video) + .map(Video::getSnippet) + .map(VideoSnippet::getPublishedAt) + .map(DateTime::getValue) + .orElse(0L); + + return to(id, GroupItemType.VIDEO, displayName, thumbUrl, channelTitle, published); + } + + public Optional from(final Channel channel) { + final String id = Optional.ofNullable(channel) + .map(Channel::getId) + .orElse(null); + final String displayName = Optional.ofNullable(channel) + .map(Channel::getSnippet) + .map(ChannelSnippet::getTitle) + .orElse(null); + final String thumbUrl = Optional.ofNullable(channel) + .map(Channel::getSnippet) + .map(ChannelSnippet::getThumbnails) + .map(ThumbnailDetails::getMedium) + .map(Thumbnail::getUrl) + .orElse(null); + final long published = Optional.ofNullable(channel) + .map(Channel::getSnippet) + .map(ChannelSnippet::getPublishedAt) + .map(DateTime::getValue) + .orElse(0L); + + return to(id, GroupItemType.CHANNEL, displayName, thumbUrl, displayName, published); + } + + public Optional from(final Playlist playlist) { + final String id = Optional.ofNullable(playlist) + .map(Playlist::getId) + .orElse(null); + final String displayName = Optional.ofNullable(playlist) + .map(Playlist::getSnippet) + .map(PlaylistSnippet::getTitle) + .orElse(null); + final String thumbUrl = Optional.ofNullable(playlist) + .map(Playlist::getSnippet) + .map(PlaylistSnippet::getThumbnails) + .map(ThumbnailDetails::getMedium) + .map(Thumbnail::getUrl) + .orElse(null); + final String channelTitle = Optional.ofNullable(playlist) + .map(Playlist::getSnippet) + .map(PlaylistSnippet::getChannelTitle) + .orElse(null); + final long published = Optional.ofNullable(playlist) + .map(Playlist::getSnippet) + .map(PlaylistSnippet::getPublishedAt) + .map(DateTime::getValue) + .orElse(0L); + + return to(id, GroupItemType.PLAYLIST, displayName, thumbUrl, channelTitle, published); + } + + /** + * @return id of video, channel, or playlist + */ + private String getIdFromResource(final ResourceId resourceId) { + return Optional.ofNullable(resourceId) + .map(resource -> Stream.of(resource.getVideoId(), resource.getChannelId(), resource.getPlaylistId())) + .orElseGet(Stream::empty) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + } + + /** + * @return id of video, channel, or playlist + */ + private GroupItemType getTypeFromResource(final ResourceId resourceId) { + return Optional.ofNullable(resourceId) + .map(resource -> { + if (resource.getVideoId() != null) { + return GroupItemType.VIDEO; + } else if (resource.getPlaylistId() != null) { + return GroupItemType.PLAYLIST; + } else if (resource.getChannelId() != null) { + return GroupItemType.CHANNEL; + } + return null; + }) + .orElse(null); + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YType.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemType.java similarity index 86% rename from src/main/java/io/mattw/youtube/commentsuite/db/YType.java rename to src/main/java/io/mattw/youtube/commentsuite/db/GroupItemType.java index 67e411b..5cb0996 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YType.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemType.java @@ -4,7 +4,7 @@ * Type definition for YouTubeObjects * */ -public enum YType { +public enum GroupItemType { UNKNOWN("Unknown"), VIDEO("Video"), CHANNEL("Channel"), @@ -13,7 +13,7 @@ public enum YType { private String display; - YType(String display) { + GroupItemType(String display) { this.display = display; } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemsTable.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemsTable.java index d869853..87bea2f 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemsTable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemsTable.java @@ -11,7 +11,7 @@ import java.util.Collections; import java.util.List; -import static io.mattw.youtube.commentsuite.CommentSuite.*; +import static io.mattw.youtube.commentsuite.CommentSuite.postEvent; import static io.mattw.youtube.commentsuite.db.SQLLoader.*; public class GroupItemsTable extends TableHelper { @@ -22,13 +22,14 @@ public GroupItemsTable(final Connection connection) { @Override public GroupItem to(ResultSet resultSet) throws SQLException { - return new GroupItem(resultSet.getString("gitem_id"), - YType.values()[resultSet.getInt("type_id") + 1], - resultSet.getString("title"), - resultSet.getString("channel_title"), - resultSet.getString("thumb_url"), - resultSet.getLong("published"), - resultSet.getLong("last_checked")); + return new GroupItem() + .setId(resultSet.getString("gitem_id")) + .setType(GroupItemType.values()[resultSet.getInt("type_id") + 1]) + .setDisplayName(resultSet.getString("title")) + .setChannelTitle(resultSet.getString("channel_title")) + .setThumbUrl(resultSet.getString("thumb_url")) + .setPublished(resultSet.getLong("published")) + .setLastChecked(resultSet.getLong("last_checked")); } @Override @@ -105,8 +106,8 @@ public void insertAll(List objects) throws SQLException { try (PreparedStatement psCG = preparedStatement(CREATE_GITEM.toString())) { for (GroupItem gi : objects) { psCG.setString(1, gi.getId()); - psCG.setInt(2, gi.getTypeId().id()); - psCG.setString(3, gi.getTitle()); + psCG.setInt(2, gi.getType().id()); + psCG.setString(3, gi.getDisplayName()); psCG.setString(4, gi.getChannelTitle()); psCG.setLong(5, gi.getPublished()); psCG.setLong(6, gi.getLastChecked()); @@ -153,7 +154,7 @@ public void update(GroupItem object) throws SQLException { public void updateAll(List objects) throws SQLException { try (PreparedStatement ps = preparedStatement(UPDATE_GITEM.toString())) { for (GroupItem item : objects) { - ps.setString(1, item.getTitle()); + ps.setString(1, item.getDisplayName()); ps.setString(2, item.getChannelTitle()); ps.setLong(3, item.getPublished()); ps.setLong(4, System.currentTimeMillis()); diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/LinkType.java b/src/main/java/io/mattw/youtube/commentsuite/db/LinkType.java new file mode 100644 index 0000000..6b10abf --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/db/LinkType.java @@ -0,0 +1,19 @@ +package io.mattw.youtube.commentsuite.db; + +public enum LinkType { + VIDEO(GroupItemType.VIDEO), + PLAYLIST(GroupItemType.PLAYLIST), + CHANNEL_USER(GroupItemType.CHANNEL), + CHANNEL_ID(GroupItemType.CHANNEL), + CHANNEL_CUSTOM(GroupItemType.CHANNEL); + + private GroupItemType itemType; + + LinkType(GroupItemType itemType) { + this.itemType = itemType; + } + + public GroupItemType getItemType() { + return itemType; + } +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/Linkable.java b/src/main/java/io/mattw/youtube/commentsuite/db/Linkable.java new file mode 100644 index 0000000..68eb403 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/db/Linkable.java @@ -0,0 +1,17 @@ +package io.mattw.youtube.commentsuite.db; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public interface Linkable { + + Map GROUP_ITEM_FORMATS = ImmutableMap.of( + GroupItemType.VIDEO, "https://youtu.be/%s", + GroupItemType.CHANNEL, "https://www.youtube.com/channel/%s", + GroupItemType.PLAYLIST, "https://www.youtube.com/playlist?list=%s" + ); + + String toYouTubeLink(); + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java index 1494272..f851b05 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java @@ -17,7 +17,7 @@ public YouTubeChannel(Channel item) { .map(ThumbnailDetails::getDefault) .map(Thumbnail::getUrl) .orElse(null)); - setTypeId(YType.CHANNEL); + setTypeId(GroupItemType.CHANNEL); } /** @@ -27,7 +27,7 @@ public YouTubeChannel(Comment item) { this(getChannelIdFromObject(item.getSnippet().getAuthorChannelId()), StringEscapeUtils.unescapeHtml4(item.getSnippet().getAuthorDisplayName()), item.getSnippet().getAuthorProfileImageUrl()); - setTypeId(YType.CHANNEL); + setTypeId(GroupItemType.CHANNEL); } /** @@ -35,7 +35,7 @@ public YouTubeChannel(Comment item) { */ public YouTubeChannel(CommentThread item) { this(item.getSnippet().getTopLevelComment()); - setTypeId(YType.CHANNEL); + setTypeId(GroupItemType.CHANNEL); } /** @@ -43,7 +43,7 @@ public YouTubeChannel(CommentThread item) { */ public YouTubeChannel(String channelId, String name, String thumbUrl) { super(channelId, name, thumbUrl); - setTypeId(YType.CHANNEL); + setTypeId(GroupItemType.CHANNEL); } public String toString() { diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java index 7ef6563..59cd380 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java @@ -42,7 +42,7 @@ public class YouTubeComment extends YouTubeObject implements Exportable { */ public YouTubeComment(Comment item, String videoId) { super(item.getId(), null, null); - this.setTypeId(YType.COMMENT); + this.setTypeId(GroupItemType.COMMENT); this.videoId = videoId; this.isReply = true; @@ -67,7 +67,7 @@ public YouTubeComment(Comment item, String videoId) { */ public YouTubeComment(CommentThread item) { super(item.getId(), null, null); - this.setTypeId(YType.COMMENT); + this.setTypeId(GroupItemType.COMMENT); this.isReply = false; this.parentId = null; @@ -91,7 +91,7 @@ public YouTubeComment(CommentThread item) { public YouTubeComment(String commentId) { super(commentId, null, null); - setTypeId(YType.COMMENT); + setTypeId(GroupItemType.COMMENT); setModerationStatus(ModerationStatus.PUBLISHED); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java index 051c756..e853a21 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java @@ -21,7 +21,7 @@ public abstract class YouTubeObject implements ImageCache, Serializable { private String thumbUrl; // Transient, we don't want this in export file. - private transient YType typeId; + private transient GroupItemType typeId; /** * This field differs from what's returned by {@link #buildYouTubeLink()} because it is used solely by @@ -30,7 +30,7 @@ public abstract class YouTubeObject implements ImageCache, Serializable { private transient String youTubeLink; public YouTubeObject() { - typeId = YType.UNKNOWN; + typeId = GroupItemType.UNKNOWN; } /** @@ -57,44 +57,49 @@ public YouTubeObject(final ResourceId id, final String title, final String thumb this.thumbUrl = thumbUrl; } - public YType getTypeId() { - return typeId; - } - - public void setTypeId(YType typeId) { - this.typeId = typeId; - } - public String getId() { return id; } - public void setId(String id) { + public YouTubeObject setId(String id) { this.id = id; + return this; } public String getTitle() { return title; } - public void setTitle(String title) { + public YouTubeObject setTitle(String title) { this.title = title; + return this; } public String getThumbUrl() { return thumbUrl; } - public void setThumbUrl(String thumbUrl) { + public YouTubeObject setThumbUrl(String thumbUrl) { this.thumbUrl = thumbUrl; + return this; + } + + public GroupItemType getTypeId() { + return typeId; + } + + public YouTubeObject setTypeId(GroupItemType typeId) { + this.typeId = typeId; + return this; } public String getYouTubeLink() { return youTubeLink; } - public void setYouTubeLink(String youTubeLink) { + public YouTubeObject setYouTubeLink(String youTubeLink) { this.youTubeLink = youTubeLink; + return this; } public String getTypeName() { diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java index 19a9c2e..aabcbd2 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java @@ -31,7 +31,7 @@ public class YouTubeVideo extends YouTubeObject implements Exportable { public YouTubeVideo(String id, String title, String thumbUrl) { super(id, title, thumbUrl); - setTypeId(YType.VIDEO); + setTypeId(GroupItemType.VIDEO); } /** @@ -41,7 +41,7 @@ public YouTubeVideo(String id, String title, String thumbUrl) { */ public YouTubeVideo(Video item) { super(item.getId(), item.getSnippet().getTitle(), item.getSnippet().getThumbnails().getMedium().getUrl()); - setTypeId(YType.VIDEO); + setTypeId(GroupItemType.VIDEO); if (item.getSnippet() != null) { VideoSnippet snippet = item.getSnippet(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java index 8b055b5..9625e53 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java @@ -6,6 +6,7 @@ import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; import io.mattw.youtube.commentsuite.db.GroupItem; +import io.mattw.youtube.commentsuite.db.GroupItemLinkResolver; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; import javafx.scene.control.*; @@ -16,8 +17,7 @@ import java.io.IOException; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -81,75 +81,48 @@ public MGMVAddItemModal(final Group group) { btnSubmit.setOnAction(ae -> new Thread(() -> { runLater(() -> btnSubmit.setDisable(true)); - Tab selectedTab = tabPane.getSelectionModel().getSelectedItem(); + final Tab selectedTab = tabPane.getSelectionModel().getSelectedItem(); + final GroupItemLinkResolver resolver = new GroupItemLinkResolver(); if (selectedTab.equals(tabSingular)) { - try { - GroupItem newItem = new GroupItem(link.getText()); - - List list = new ArrayList<>(); - list.add(newItem); - - if (!list.isEmpty()) { - try { - database.groupItems().insertAll(this.group, list); - database.commit(); - runLater(() -> btnClose.fire()); - } catch (SQLException e1) { - runLater(() -> setError(e1.getClass().getSimpleName())); - } + final Optional newItem = resolver.from(link.getText()); + if (newItem.isPresent()) { + try { + database.groupItems().insertAll(this.group, Collections.singletonList(newItem.get())); + database.commit(); + runLater(() -> btnClose.fire()); + } catch (SQLException e1) { + runLater(() -> setError(e1.getClass().getSimpleName())); } - } catch (IOException e) { - String message = e.getMessage(); - + } else { + String message = "There was a problem, could not resolve link."; runLater(() -> setError(message)); - - logger.error("Failed to submit link", e); + logger.error(message); } } else if (selectedTab.equals(tabBulk)) { - List givenLinks = Stream.of(multiLink.getText().split("\n")) + final List uniqueLinks = Stream.of(multiLink.getText().split("\n")) .map(String::trim) - .filter(item -> StringUtils.isNotEmpty(item) && // not empty - item.toLowerCase().startsWith("http") && // is a URL - item.toLowerCase().contains("youtu")) // most likely a youtu.be / youtube.com link - .distinct() // remove duplicates + .filter(StringUtils::isNotBlank) + .distinct() .collect(Collectors.toList()); - if (!givenLinks.isEmpty()) { - configData.setFastGroupAdd(fastGroupAdd.isSelected()); - configFile.save(); - - List list = new ArrayList<>(); - - int failures = 0; - - for (String givenLink : givenLinks) { - try { - GroupItem newItem = new GroupItem(givenLink, configData.isFastGroupAdd()); - - list.add(newItem); - } catch (IOException e) { - failures++; - } - } + if (uniqueLinks.isEmpty()) { + return; + } - if (!list.isEmpty()) { - try { - database.groupItems().insertAll(this.group, list); - database.commit(); - runLater(() -> btnClose.fire()); - } catch (SQLException e1) { - failures++; - } + final List resolvedItems = resolver.from(uniqueLinks); + if (!resolvedItems.isEmpty()) { + try { + database.groupItems().insertAll(this.group, resolvedItems); + database.commit(); + runLater(() -> btnClose.fire()); + } catch (SQLException e1) { + runLater(() -> setError(e1.getClass().getSimpleName())); } - - logger.warn("Failed to parse or insert {} links for bulk add", failures); } else { - String message = "No valid links to submit (bulk)."; - + String message = "There was a problem, could not resolve links."; runLater(() -> setError(message)); - - logger.info(message); + logger.error(message); } } diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java index 265ec21..969c098 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java @@ -40,20 +40,20 @@ public MGMVGroupItemView(final GroupItem groupItem) { icon.setManaged(false); icon.setVisible(false); } else { - icon.setImage(groupItem.getDefaultThumb()); + icon.setImage(ImageCache.findOrGetImage(groupItem.getId(), groupItem.getThumbUrl())); icon.setManaged(true); icon.setVisible(true); } - title.setText(groupItem.getTitle()); + title.setText(groupItem.getDisplayName()); author.setText(groupItem.getChannelTitle()); - type.setText(groupItem.getTypeName()); + type.setText(groupItem.getType().getDisplay()); icon.setCursor(Cursor.HAND); - icon.setOnMouseClicked(me -> browserUtil.open(groupItem.buildYouTubeLink())); + icon.setOnMouseClicked(me -> browserUtil.open(groupItem.toYouTubeLink())); new Thread(() -> { - Image image = ImageCache.findOrGetImage(groupItem); + Image image = ImageCache.findOrGetImage(groupItem.getId(), groupItem.getThumbUrl()); runLater(() -> icon.setImage(image)); }).start(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java index 0eddb0c..7173ed8 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java @@ -64,8 +64,6 @@ public void initialize(URL location, ResourceBundle resources) { SelectionModel selectionModel = comboGroupSelect.getSelectionModel(); selectionModel.selectedItemProperty().addListener((o, ov, nv) -> { - logger.debug("selectedItemProperty({}, {}, {})", ov, nv, selectionModel.getSelectedIndex()); - if (nv != null) { ManageGroupsManager manager = managerCache.getIfPresent(nv.getGroupId()); if (manager != null) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java index 3938caf..e9b32b4 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java @@ -141,7 +141,7 @@ public void updateSelectionLabel() { btnReset.setDisable(selectedVideo == null); lblSelection.setText(String.format("%s > %s > %s", group != null ? group.getName() : "$group", - groupItem != null ? groupItem.getTitle() : "$groupItem", + groupItem != null ? groupItem.getDisplayName() : "$groupItem", selectedVideo != null ? selectedVideo.getTitle() : ALL_VIDEOS)); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java index f758050..a765f24 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java @@ -2,9 +2,7 @@ import com.google.common.eventbus.Subscribe; import io.mattw.youtube.commentsuite.CommentSuite; -import io.mattw.youtube.commentsuite.db.CommentDatabase; -import io.mattw.youtube.commentsuite.db.Group; -import io.mattw.youtube.commentsuite.db.GroupItem; +import io.mattw.youtube.commentsuite.db.*; import io.mattw.youtube.commentsuite.events.GroupAddEvent; import io.mattw.youtube.commentsuite.events.GroupDeleteEvent; import io.mattw.youtube.commentsuite.events.GroupRenameEvent; @@ -22,6 +20,7 @@ import java.sql.SQLException; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import static javafx.application.Platform.runLater; @@ -109,18 +108,12 @@ public SYAddToGroupModal(final ListView listView) { } private void submitItemsToGroup(final List items, final Group group) { - List list = items.stream() - .map(SearchYouTubeListItem::getYoutubeURL) - .map(link -> { - try { - return new GroupItem(link); - } catch (IOException e) { - logger.error("Failed to parse to GroupItem", e); - - return null; - } - }) - .filter(Objects::nonNull) + final GroupItemResolver resolver = new GroupItemResolver(); + final List list = items.stream() + .map(SearchYouTubeListItem::getData) + .map(resolver::from) + .filter(Optional::isPresent) + .map(Optional::get) .collect(Collectors.toList()); logger.debug("Group Items to add [list={}]", list.toString()); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java index 8c59265..260a2a1 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java @@ -709,7 +709,9 @@ private void reloadGroupItems() { final Group selectedGroup = comboGroupSelect.getValue(); final List groupItems = database.groupItems().byGroup(selectedGroup); - final GroupItem all = new GroupItem(GroupItem.ALL_ITEMS, String.format("All Items (%s)", groupItems.size())); + final GroupItem all = new GroupItem() + .setId(GroupItem.ALL_ITEMS) + .setDisplayName(String.format("All Items (%s)", groupItems.size())); runLater(() -> { comboGroupItemSelect.getItems().clear(); comboGroupItemSelect.getItems().add(all); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java index b3bc9fb..5e16613 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java @@ -5,7 +5,7 @@ import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.GroupItem; -import io.mattw.youtube.commentsuite.db.YType; +import io.mattw.youtube.commentsuite.db.GroupItemType; import io.mattw.youtube.commentsuite.util.ExecutorGroup; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -49,12 +49,12 @@ private void produce() { } try { - final YType type = item.getTypeId(); - if (type == YType.CHANNEL) { + final GroupItemType type = item.getType(); + if (type == GroupItemType.CHANNEL) { fromChannel(item); - } else if (type == YType.PLAYLIST) { + } else if (type == GroupItemType.PLAYLIST) { fromPlaylist(item); - } else if (type == YType.VIDEO) { + } else if (type == GroupItemType.VIDEO) { fromVideo(item); } diff --git a/src/main/resources/io/mattw/youtube/commentsuite/fxml/MainQuotaModal.fxml b/src/main/resources/io/mattw/youtube/commentsuite/fxml/MainQuotaModal.fxml index 9358c12..5b9b6e1 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/fxml/MainQuotaModal.fxml +++ b/src/main/resources/io/mattw/youtube/commentsuite/fxml/MainQuotaModal.fxml @@ -14,10 +14,8 @@