一些优化，修复可能导致panic的代码

cmd/webscan.go

@@ -78,7 +78,7 @@ var webScanCmd = &cobra.Command{
             // 初始化爬虫
             crawler.NewCrawlergo(show)
             for _, target := range conf.GlobalConfig.Options.Targets {
-                mode.Active(target)
+                mode.Active(target, nil)
             }
 
             if copilot { // 阻塞，不退出

conf/banner.go

@@ -17,4 +17,4 @@ var Banner = `
 
 const Website = "https://github.com/yhy0/Jie"
 
-const Version = "1.0.2"
+const Version = "1.0.4"

pkg/mode/active.go

@@ -35,12 +35,12 @@ var extensionFilter = []string{
 }
 
 // Active 主动扫描 调用爬虫扫描, 只会输入一个域名
-func Active(target string) ([]string, []string) {
+func Active(target string, fingerprint []string) ([]string, []string) {
     if target == "" {
         logging.Logger.Errorln("target must be set")
         return nil, nil
     }
-
+    
     // 判断是否以 http https 开头
     httpMatch, _ := regexp.MatchString("^(http)s?://", target)
     if !httpMatch {
@@ -51,7 +51,7 @@ func Active(target string) ([]string, []string) {
             target = fmt.Sprintf("http://%s", target)
         }
     }
-
+    
     parseUrl, err := url.Parse(target)
     if err != nil {
         logging.Logger.Errorln(err)
@@ -64,12 +64,12 @@ func Active(target string) ([]string, []string) {
     } else {
         host = parseUrl.Host
     }
-
+    
     t := &task.Task{
         Parallelism: conf.Parallelism + 1,
         ScanTask:    make(map[string]*task.ScanTask),
     }
-
+    
     client := httpx.NewClient(nil)
     t.ScanTask[host] = &task.ScanTask{
         PerServer: make(map[string]bool),
@@ -80,71 +80,71 @@ func Active(target string) ([]string, []string) {
         // TODO 更优雅的实现方式
         Wg: sizedwaitgroup.New(3 + 3),
     }
-
+    
     t.Wg = sizedwaitgroup.New(t.Parallelism)
-
+    
     // 爬虫前，进行连接性、指纹识别、 waf 探测
     resp, err := client.Request(target, "GET", "", nil)
     if err != nil {
         logging.Logger.Errorln("End: ", err)
         return nil, nil
     }
-
+    
     technologies := fingprints.Identify([]byte(resp.Body), resp.Header)
-
+    
     wafs := waf.Scan(target, resp.Body, client)
-
+    
     // 爬虫的同时进行指纹识别
-    subdomains := Crawler(target, wafs, t)
-
+    subdomains := Crawler(target, wafs, t, fingerprint)
+    
     t.Wg.Wait()
-
+    
     logging.Logger.Debugln("Fingerprints: ", t.Fingerprints)
-
+    
     t.Fingerprints = funk.UniqString(append(t.Fingerprints, technologies...))
-
+    
     return subdomains, t.Fingerprints
 }
 
 // Crawler 运行爬虫, 对爬虫结果进行处理
-func Crawler(target string, waf []string, t *task.Task) []string {
+func Crawler(target string, waf []string, t *task.Task, fingerprint []string) []string {
     t.Wg.Add()
     defer t.Wg.Done()
     var targets []*model.Request
-
+    
     var req model.Request
     u, err := model.GetUrl(target)
     if err != nil {
         logging.Logger.Error("parse url failed, ", err)
         return nil
     }
-
+    
     req = model.GetRequest(config.GET, u, getOption())
     req.Proxy = crawler.TaskConfig.Proxy
     targets = append(targets, &req)
-
+    
     if len(targets) != 0 {
         logging.Logger.Infof("Init crawler task, host: %s, max tab count: %d, max crawl count: %d, max runtime: %d %s", targets[0].URL.Host, crawler.TaskConfig.MaxTabsCount, crawler.TaskConfig.MaxCrawlCount, crawler.TaskConfig.MaxRunTime, "s")
     } else {
         logging.Logger.Errorln("no validate target.")
         return nil
     }
-
+    
     if crawler.TaskConfig.Proxy != "" {
         logging.Logger.Info("request with proxy: ", crawler.TaskConfig.Proxy)
     }
-
+    
     // 实时获取结果
     onResult := func(result *crawlergo.OutResult) {
         // 不对这些进行漏扫
         for _, suffix := range extensionFilter {
-            if strings.HasSuffix(result.ReqList.URL.String(), suffix) {
+            if strings.HasSuffix(result.ReqList.URL.Path, suffix) {
                 return
             }
         }
-
+        
         logging.Logger.Infof("[crawlergo]: [%s] %v %v", result.ReqList.Method, result.ReqList.URL.String(), result.ReqList.PostData)
-
+        
         curl := strings.ReplaceAll(result.ReqList.URL.String(), "\\n", "")
         curl = strings.ReplaceAll(curl, "\\t", "")
         curl = strings.ReplaceAll(curl, "\\n", "")
@@ -153,7 +153,7 @@ func Crawler(target string, waf []string, t *task.Task) []string {
             logging.Logger.Errorln(err)
             return
         }
-
+        
         var host string
         // 有的会带80、443端口号，导致    example.com 和 example.com:80、example.com:443被认为是不同的网站
         if strings.Contains(parseUrl.Host, ":443") || strings.Contains(parseUrl.Host, ":80") {
@@ -166,45 +166,50 @@ func Crawler(target string, waf []string, t *task.Task) []string {
             logging.Logger.Errorln(err)
             return
         }
-
+
+        _fingerprint := fingprints.Identify([]byte(resp.Body), resp.Header)
+
+        fingerprint = funk.UniqString(append(_fingerprint, fingerprint...))
+
         // 对爬虫结果格式化
         var crawlResult = &input.CrawlResult{
-            Url:         curl,
-            ParseUrl:    parseUrl,
-            Host:        host,
-            Target:      target,
-            Method:      result.ReqList.Method,
-            Source:      result.ReqList.Source,
-            Headers:     make(map[string]string),
-            RequestBody: result.ReqList.PostData,
-            Waf:         waf,
-            Resp:        resp,
-            UniqueId:    util.UUID(), // 这里爬虫中已经判断过了，所以生成一个 uuid 就行
+            Url:          curl,
+            ParseUrl:     parseUrl,
+            Host:         host,
+            Target:       target,
+            Method:       result.ReqList.Method,
+            Source:       result.ReqList.Source,
+            Headers:      make(map[string]string),
+            RequestBody:  result.ReqList.PostData,
+            Fingerprints: fingerprint,
+            Waf:          waf,
+            Resp:         resp,
+            UniqueId:     util.UUID(), // 这里爬虫中已经判断过了，所以生成一个 uuid 就行
         }
-
+        
         // 分发扫描任务
         go t.Distribution(crawlResult)
     }
-
+    
     // 开始爬虫任务
     crawlerTask, err := crawlergo.NewCrawlerTask(targets, crawler.TaskConfig, onResult)
     if err != nil {
         logging.Logger.Error("create crawler task failed.")
         return nil
     }
-
+    
     crawlerTask.Browser = crawler.Browser
-
+    
     crawlerTask.Run()
-
+    
     logging.Logger.Infof("Task finished, %d results, %d subdomains found, runtime: %d s", len(crawlerTask.Result.ReqList), len(crawlerTask.Result.SubDomainList), time.Now().Unix()-crawlerTask.Start.Unix())
-
+    
     return crawlerTask.Result.SubDomainList
 }
 
 func getOption() model.Options {
     var option model.Options
-
+    
     if crawler.TaskConfig.ExtraHeadersString != "" {
         err := json.Unmarshal([]byte(crawler.TaskConfig.ExtraHeadersString), &crawler.TaskConfig.ExtraHeaders)
         if err != nil {