数据包过滤排除和去重逻辑更新

src/config/ConfigPanel.java

@@ -45,6 +45,9 @@ public class ConfigPanel extends JPanel{
 	public static JRadioButton ignoreHTTPStaus500;
 	public static JRadioButton ignoreHTTPStaus400;
 	public static JRadioButton ignoreWrongCAHost;
+
+	public static JRadioButton removeItemIfIgnored;
+
 	public static JRadioButton DisplayContextMenuOfBurp;
 	public static JRadioButton rdbtnSaveTrafficTo;
 	public static JTextField textFieldPortScanner;
@@ -297,15 +300,18 @@ public void actionPerformed(ActionEvent e) {
 		ignoreHTTPStaus500.setSelected(true);
 
 
-		ignoreHTTPStaus400 = new JRadioButton("Ignore http Status 400(The plain HTTP request was sent to HTTPS port)");
+		ignoreHTTPStaus400 = new JRadioButton("Ignore items that http Status is 400(The plain HTTP request was sent to HTTPS port)");
 		ignoreHTTPStaus400.setSelected(true);
 
 
 
-		ignoreWrongCAHost = new JRadioButton("Ignore Host that IP Address and Certificate Authority not match");
+		ignoreWrongCAHost = new JRadioButton("Ignore items that IP Address and Certificate Authority do not match");
 		ignoreWrongCAHost.setSelected(false);
 
 
+		removeItemIfIgnored = new JRadioButton("Remove item if ignored(Marked as check done by default)");
+		removeItemIfIgnored.setSelected(true);
+
 
 		rdbtnSaveTrafficTo = new JRadioButton("Save traffic to Elastic");
 		rdbtnSaveTrafficTo.setSelected(false);
@@ -414,6 +420,7 @@ public void actionPerformed(ActionEvent e) {
 		add(ignoreHTTPStaus500, new MyGridBagLayout(++rowIndex,2));
 		add(ignoreHTTPStaus400, new MyGridBagLayout(++rowIndex,2));
 		add(ignoreWrongCAHost, new MyGridBagLayout(++rowIndex,2));
+		add(removeItemIfIgnored,new MyGridBagLayout(++rowIndex,2));
 		add(rdbtnSaveTrafficTo, new MyGridBagLayout(++rowIndex,2));
 
 	}

src/config/LineConfig.java

@@ -325,6 +325,22 @@ public  static LineConfig FromJson(String instanceString) {// throws Exception {
 		//return new Gson().fromJson(instanceString, LineConfig.class);
 	}
 
+	/**
+	 *是否是从http跳转到相同URL的https 
+	 **/
+	public static boolean isRedirectToHttps(LineEntry item) {
+		if (item.getProtocol().equalsIgnoreCase("http")) {
+			if (400>item.getStatuscode() && item.getStatuscode() >=300) {
+				String locationUrl = item.getHeaderValueOf(false,"Location");
+				locationUrl = locationUrl.toLowerCase().replace("https://", "http://");
+				if (locationUrl.equalsIgnoreCase(item.getUrl())) {
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
 	/**
 	 * 
 	 * 同一个主机的多个请求，可以根据某些条件丢弃一些。比如
@@ -341,23 +357,52 @@ public static List<LineEntry> doSameHostFilter(List<LineEntry> entries) {
 
 		LineEntry httpsOk =null;
 		LineEntry httpOk =null;
+
+		LineEntry otherPorthttpsOk =null;
+		LineEntry otherPorthttpOk =null;
+
 		for (LineEntry item:entries) {
-			if (item.getProtocol().equalsIgnoreCase("https") && item.getPort()==443 && item.getStatuscode()>0) {
-				httpsOk = item;
-			}
+			if (item.getPort()==443 ||item.getPort()==80) {
+				if (item.getProtocol().equalsIgnoreCase("https")  && item.getStatuscode()>0) {
+					httpsOk = item;
+				}
 
-			if (item.getProtocol().equalsIgnoreCase("http") && item.getPort()==80 && item.getStatuscode()>0) {
-				httpOk = item;
+				if (item.getProtocol().equalsIgnoreCase("http") && item.getStatuscode()>0) {
+					httpOk = item;
+					if (isRedirectToHttps(item)) {
+						httpOk = null;
+					}
+				}
+			}else {
+				if (item.getProtocol().equalsIgnoreCase("https") && item.getStatuscode()>0) {
+					otherPorthttpsOk = item;
+				}
+
+				if (item.getProtocol().equalsIgnoreCase("http") && item.getStatuscode()>0) {
+					otherPorthttpOk = item;
+					if (isRedirectToHttps(item)) {
+						otherPorthttpOk = null;
+					}
+				}
 			}
 		}
 
 		if (httpsOk !=null && httpOk !=null ) {
 			if (ConfigPanel.ignoreHTTP.isSelected()) {
-				entries.remove(httpOk);
+				httpOk.setCheckStatus(LineEntry.CheckStatus_Checked);
 			}else if (ConfigPanel.ignoreHTTPS.isSelected()) {
-				entries.remove(httpsOk);
+				httpsOk.setCheckStatus(LineEntry.CheckStatus_Checked);
 			}
 		}
+
+		if (otherPorthttpsOk !=null && otherPorthttpOk !=null ) {
+			if (ConfigPanel.ignoreHTTP.isSelected()) {
+				otherPorthttpOk.setCheckStatus(LineEntry.CheckStatus_Checked);
+			}else if (ConfigPanel.ignoreHTTPS.isSelected()) {
+				otherPorthttpsOk.setCheckStatus(LineEntry.CheckStatus_Checked);
+			}
+		}
+
 		return entries;
 	}
 	/*
@@ -389,6 +434,12 @@ public static LineEntry doFilter(LineEntry entry) {
 			return entry;
 		}
 
+		if (entry.getStatuscode() == 403 && entry.getTitle().equals("Direct IP access not allowed | Cloudflare")) {
+			stdout.println(String.format("--- [%s] --- Direct Cloudflare IP access",entry.getUrl()));
+			entry.setCheckStatus(LineEntry.CheckStatus_Checked);
+			return entry;
+		}
+
 		//<head><title>403 Forbidden</title></head>
 		/*
 		if (entry.getStatuscode() == 403 && entry.getTitle().equals("403 Forbidden")){
@@ -401,11 +452,12 @@ public static LineEntry doFilter(LineEntry entry) {
 		}*/
 
 		//<title>Welcome to nginx!</title>
+		/*
 		if (entry.getStatuscode() == 200 && entry.getTitle().equals("Welcome to nginx!")
 				&& entry.getContentLength()<=612 ){
 			entry.setCheckStatus(LineEntry.CheckStatus_Checked);
 			return entry;
-		}
+		}*/
 
 		/*
 		if (null != blacklistStatusCodeSet && blacklistStatusCodeSet.size()>0) {

src/test/URLTest.java

@@ -0,0 +1,21 @@
+
+import java.net.URL;
+
+public class URLTest {
+    public static void main(String[] args) {
+        try {
+            // 创建URL对象
+            URL url = new URL("https://www.example.com/path/to/resource?param1=value1#111111");
+
+            // 使用 getFile() 方法获取文件名,带查询参数
+            String file = url.getFile();
+            System.out.println("File: " + file);
+
+            // 使用 getPath() 方法获取路径，不带查询参数
+            String path = url.getPath();
+            System.out.println("Path: " + path);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

src/thread/Producer.java

@@ -13,6 +13,7 @@
 import burp.BurpExtender;
 import burp.IBurpExtenderCallbacks;
 import burp.IExtensionHelpers;
+import config.ConfigPanel;
 import config.LineConfig;
 import title.LineEntry;
 import utils.DomainToURLs;
@@ -78,6 +79,9 @@ public void run() {
 					LineEntry item = new LineEntry(Url).firstRequest(guiMain.getTitlePanel().getTempConfig());
 
 					LineConfig.doFilter(item);
+					if (ConfigPanel.removeItemIfIgnored.isSelected() && item.getCheckStatus().equals(LineEntry.CheckStatus_Checked)) {
+						continue;
+					}
 					item.setEntrySource(type);
 
 					String url = item.getUrl();
@@ -105,6 +109,9 @@ public void run() {
 				tempEntries = LineConfig.doSameHostFilter(tempEntries);
 
 				for (LineEntry item:tempEntries) {
+					if (ConfigPanel.removeItemIfIgnored.isSelected() && item.getCheckStatus().equals(LineEntry.CheckStatus_Checked)) {
+						continue;
+					}
 					guiMain.getTitlePanel().getTitleTable().getLineTableModel().addNewLineEntry(item);
 					//stdout.println(new LineEntry(messageinfo,true).ToJson());
 					int leftTaskNum = domainQueue.size();

src/title/GetTitleTempConfig.java

@@ -33,18 +33,11 @@ public void setThreadNumber(int threadNumber) {
 		this.threadNumber = threadNumber;
 	}
 
-	@Deprecated
-	GetTitleTempConfig(){
-		handlePriavte = WetherHandlePrivate();
-		cookie = inputCookie();
-		threadNumber = inputThreadNumber(50);
-	}
-
 	GetTitleTempConfig(int domainNum){
 		handlePriavte = WetherHandlePrivate();
 		cookie = inputCookie();
-		int num = threadNumberShouldUse(domainNum);
-		threadNumber = inputThreadNumber(num);
+		threadNumber = threadNumberShouldUse(domainNum);
+		//threadNumber = inputThreadNumber(num);
 	}
 
 	private static boolean WetherHandlePrivate() {
@@ -63,6 +56,7 @@ public static String inputCookie() {
 		return cookie;
 	}
 
+	@Deprecated
 	public static int inputThreadNumber(int defaultThreadNum) {
 		int times = 3;
 		while (times > 0) {

src/title/LineEntryMenu.java

@@ -847,7 +847,13 @@ public void actionPerformed(ActionEvent actionEvent) {
 		JMenuItem batchClearCommentsItem = new JMenuItem(new AbstractAction("Clear Comments") {
 			@Override
 			public void actionPerformed(ActionEvent actionEvent) {
-				titlepanel.getTitleTable().getLineTableModel().clearComments(modelRows);
+				new SwingWorker(){
+					@Override
+					protected Object doInBackground() throws Exception {
+						titlepanel.getTitleTable().getLineTableModel().clearComments(modelRows);
+						return null;
+					}
+				}.execute();
 			}
 		});
 

src/title/LineTableModel.java

@@ -8,6 +8,7 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -702,10 +703,6 @@ public void findAndMarkDuplicateItems() {
 		for (int i=lineEntries.size()-1;i>=0 ;i-- ) {//降序删除才能正确删除每个元素
 			try {
 				LineEntry entry = lineEntries.get(i);
-				if (entry == null) {
-					throw new ArrayIndexOutOfBoundsException("can't find item with index "+i);
-				}
-
 				markFullSameEntries(entry);
 
 			} catch (Exception e) {
@@ -1044,41 +1041,49 @@ public void freshAllASNInfo(){
 
 	/**
 	 * 查找完全一模一样的数据包（带时间戳锚点的URL可以不同）
+	 * 
+	 * 相同IP和端口,URL path下，即使域名不同，返回包不同（页面包含随机js、css链接），只要status和length相同，就是重复的web服务了
 	 */
 	public void markFullSameEntries(LineEntry entry) {//
 		if (lineEntries == null) return;
 
 		for (int i=lineEntries.size()-1;i>=0 ;i-- ) {//降序删除才能正确删除每个元素
 			LineEntry value = lineEntries.get(i);
 
+			if (entry.getComments().contains("duplicateItem")) {
+				//已经被标注过，不用再找它的相同项了
+				continue;
+			}
+
 			if (value.equals(entry)){
 				continue;//首先得排除自己，否则删除时就全删除了
 			}
 
-			if (!value.getUrl().equals(entry.getUrl())){
-				continue;
-			}
-			if (!value.getRequest().equals(entry.getRequest())){
+			if (value.getStatuscode()!=entry.getStatuscode()){
 				continue;
 			}
-			if (!value.getResponse().equals(entry.getResponse())){
+
+			if (value.getContentLength()!=entry.getContentLength()){
 				continue;
 			}
-			if (!value.getComments().equals(entry.getComments())){
+
+			if (value.getPort()!=entry.getPort()){
 				continue;
 			}
 
-			if (!value.getIPSet().equals(entry.getIPSet()) && !value.getIPSet().isEmpty() && !entry.getIPSet().isEmpty()){
+			if (!value.getIPSet().equals(entry.getIPSet())|| value.getIPSet().isEmpty()){
 				//只有当IP不为空才有比较的必要
 				continue;
 			}
 
-			if (!value.getCNAMESet().equals(entry.getCNAMESet()) && !value.getCNAMESet().isEmpty() && !entry.getCNAMESet().isEmpty() ){
-				//只有当CNAME不为空才有比较的必要
+			String url1 = value.getUrl().replaceFirst(value.getHost(),"");
+			String url2 = entry.getUrl().replaceFirst(entry.getHost(),"");
+			if (!Objects.equals(url1, url2)){
 				continue;
 			}
 
 			value.addComment("duplicateItem");
+			value.setCheckStatus(LineEntry.CheckStatus_Checked);
 			fireTableRowsUpdated(i,i);//主动通知更新，否则不会写入数据库!!!
 		}
 	}