(feat) add flag to save response content

fixes #190

cmd/scan.go

@@ -33,8 +33,10 @@ screenshots (still recording what gowitness could get using a --writer-*), use
 "gorod". If you prefer a *much* better chance of having a screenshot taken, use
 "chromedp" (the default). The "chromedp" driver tradeoff is resource usage, however.
 `),
-	Example: `  Scan targets from a file, dont prepend http:// to URI targets and filter by port 80:
+	Example: `  Scan targets from a nessus results file, dont prepend http:// to URI targets and filter by port 80:
    $ gowitness scan nessus -f ./scan-results.nessus --port 80
+  Scan a targets from a file, skipping http urls and storing network request content as well:
+   $ gowitness scan file -f ~/targets.txt --no-http --save-content
   Scan a CIDR, logging scan errors (can be verbose!) and using 20 'threads':
    $ gowitness scan cidr -t 20 --log-scan-errors -c 10.20.20.0/28
   Scan a single target, writing results to a SQLite database and JSON lines:
@@ -128,6 +130,7 @@ func init() {
 	scanCmd.PersistentFlags().BoolVar(&opts.Scan.ScreenshotFullPage, "screenshot-fullpage", false, "Do fullpage screenshots, instead of just the viewport")
 	scanCmd.PersistentFlags().StringVar(&opts.Scan.JavaScript, "javascript", "", "A JavaScript function to evaluate on every page, before a screenshot. Note: It must be a JavaScript function! eg: () => console.log('gowitness');")
 	scanCmd.PersistentFlags().StringVar(&opts.Scan.JavaScriptFile, "javascript-file", "", "A file containing a JavaScript function to evaluate on every page, before a screenshot. See --javascript")
+	scanCmd.PersistentFlags().BoolVar(&opts.Scan.SaveContent, "save-content", false, "Save content from network requests to the configured writers. WARNING: This flag has the potential to make your storage explode in size")
 
 	// chrome options
 	scanCmd.PersistentFlags().StringVar(&opts.Chrome.Path, "chrome-path", "", "The path to a Google Chrome binary to use (downloads a platform appropriate binary by default)")

pkg/models/models.go

@@ -62,7 +62,7 @@ type TLS struct {
 	KeyExchange              string       `json:"key_exchange"`
 	Cipher                   string       `json:"cipher"`
 	SubjectName              string       `json:"subject_name"`
-	SanList                  []TLSSanList `json:"san_list"`
+	SanList                  []TLSSanList `json:"san_list" gorm:"constraint:OnDelete:CASCADE"`
 	Issuer                   string       `json:"issuer"`
 	ValidFrom                time.Time    `json:"valid_from"`
 	ValidTo                  time.Time    `json:"valid_to"`
@@ -102,6 +102,7 @@ type NetworkLog struct {
 	RemoteIP    string      `json:"remote_ip"`
 	MIMEType    string      `json:"mime_type"`
 	Time        time.Time   `json:"time"`
+	Content     []byte      `json:"content"`
 	Error       string      `json:"error"`
 }
 

pkg/runner/drivers/chromedp.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/cdproto/page"
 	"github.com/chromedp/cdproto/runtime"
@@ -272,9 +273,30 @@ func (run *Chromedp) Witness(target string, runner *runner.Runner) (*models.Resu
 
 				// write the network log
 				resultMutex.Lock()
+				entryIndex := len(result.Network)
 				result.Network = append(result.Network, entry)
 				resultMutex.Unlock()
 
+				// if we need to write the body, do that
+				// https://github.com/chromedp/chromedp/issues/543
+				if run.options.Scan.SaveContent {
+					go func(index int) {
+						c := chromedp.FromContext(navigationCtx)
+						p := network.GetResponseBody(e.RequestID)
+						body, err := p.Do(cdp.WithExecutor(navigationCtx, c.Target))
+						if err != nil {
+							if run.options.Logging.LogScanErrors {
+								run.log.Error("could not get network request response body", "url", e.Response.URL, "err", err)
+								return
+							}
+						}
+
+						resultMutex.Lock()
+						result.Network[index].Content = body
+						resultMutex.Unlock()
+
+					}(entryIndex)
+				}
 			}
 		// mark a request as failed
 		case *network.EventLoadingFailed:

pkg/runner/drivers/go-rod.go

@@ -269,8 +269,26 @@ func (run *Gorod) Witness(target string, runner *runner.Runner) (*models.Result,
 
 				// write the network log
 				resultMutex.Lock()
+				entryIndex := len(result.Network)
 				result.Network = append(result.Network, entry)
 				resultMutex.Unlock()
+
+				// if we need to write the body, do that
+				if run.options.Scan.SaveContent {
+					go func(index int) {
+						body, err := proto.NetworkGetResponseBody{RequestID: e.RequestID}.Call(page)
+						if err != nil {
+							if run.options.Logging.LogScanErrors {
+								run.log.Error("could not get network request response body", "url", e.Response.URL, "err", err)
+								return
+							}
+						}
+
+						resultMutex.Lock()
+						result.Network[index].Content = []byte(body.Body)
+						resultMutex.Unlock()
+					}(entryIndex)
+				}
 			}
 
 			return dismissEvents

pkg/runner/options.go

@@ -76,6 +76,9 @@ type Scan struct {
 	// JavaScript to evaluate on every page
 	JavaScript     string
 	JavaScriptFile string
+	// Save content stores content from network requests (warning) this
+	// could make written artefacts huge
+	SaveContent bool
 }
 
 // NewDefaultOptions returns Options with some default values

web/ui/src/lib/api/types.ts

@@ -96,6 +96,7 @@ interface networklog {
   mime_type: string;
   time: string;
   error: string;
+  content: string;
 }
 
 interface consolelog {

web/ui/src/pages/detail/Detail.tsx

@@ -4,7 +4,7 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { ScrollArea } from "@/components/ui/scroll-area";
-import { ExternalLink, ChevronLeft, ChevronRight, Code, ClockIcon, Trash2Icon } from 'lucide-react';
+import { ExternalLink, ChevronLeft, ChevronRight, Code, ClockIcon, Trash2Icon, DownloadIcon } from 'lucide-react';
 import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, } from "@/components/ui/dialog";
 import { WideSkeleton } from '@/components/loading';
 import { Form, Link, useParams } from 'react-router-dom';
@@ -38,6 +38,29 @@ const ScreenshotDetailPage = () => {
     return "bg-blue-500 text-white";
   };
 
+  const handleDownload = (base64Content: string, url: string) => {
+    const binaryString = atob(base64Content);
+
+    // Create an array of 8-bit unsigned integers from the binary string
+    const binaryLength = binaryString.length;
+    const bytes = new Uint8Array(binaryLength);
+    for (let i = 0; i < binaryLength; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+
+    const blob = new Blob([bytes], { type: 'application/octet-stream' });
+    const link = document.createElement('a');
+    link.href = window.URL.createObjectURL(blob);
+
+    // Extract the last part of the URL for the filename
+    const urlParts = new URL(url).pathname.split('/');
+    const fileName = urlParts[urlParts.length - 1] || 'download';
+    link.download = fileName;
+
+    link.click();
+    window.URL.revokeObjectURL(link.href);
+  };
+
   if (loading) return <WideSkeleton />;
   if (!detail) return;
 
@@ -285,6 +308,7 @@ const ScreenshotDetailPage = () => {
                   <TableRow>
                     <TableHead>HTTP</TableHead>
                     <TableHead></TableHead>
+                    <TableHead></TableHead>
                     <TableHead>URL</TableHead>
                   </TableRow>
                 </TableHeader>
@@ -313,6 +337,16 @@ const ScreenshotDetailPage = () => {
                           </Tooltip>
                         </TooltipProvider>
                       </TableCell>
+                      <TableCell>
+                        {log.content && log.content.length > 0 && (
+                          <div
+                            className="cursor-pointer"
+                            onClick={() => handleDownload(log.content, log.url)}
+                          >
+                            <DownloadIcon className="w-3 h-3" />
+                          </div>
+                        )}
+                      </TableCell>
                       <TableCell
                         className="break-all cursor-pointer"
                         onClick={() => copyToClipboard(log.url, 'URL')}