diff --git a/cmd/scan.go b/cmd/scan.go
index a2a021a5..ec857081 100644
--- a/cmd/scan.go
+++ b/cmd/scan.go
@@ -33,8 +33,10 @@ screenshots (still recording what gowitness could get using a --writer-*), use
"gorod". If you prefer a *much* better chance of having a screenshot taken, use
"chromedp" (the default). The "chromedp" driver tradeoff is resource usage, however.
`),
- Example: ` Scan targets from a file, dont prepend http:// to URI targets and filter by port 80:
+ Example: ` Scan targets from a nessus results file, dont prepend http:// to URI targets and filter by port 80:
$ gowitness scan nessus -f ./scan-results.nessus --port 80
+ Scan a targets from a file, skipping http urls and storing network request content as well:
+ $ gowitness scan file -f ~/targets.txt --no-http --save-content
Scan a CIDR, logging scan errors (can be verbose!) and using 20 'threads':
$ gowitness scan cidr -t 20 --log-scan-errors -c 10.20.20.0/28
Scan a single target, writing results to a SQLite database and JSON lines:
@@ -128,6 +130,7 @@ func init() {
scanCmd.PersistentFlags().BoolVar(&opts.Scan.ScreenshotFullPage, "screenshot-fullpage", false, "Do fullpage screenshots, instead of just the viewport")
scanCmd.PersistentFlags().StringVar(&opts.Scan.JavaScript, "javascript", "", "A JavaScript function to evaluate on every page, before a screenshot. Note: It must be a JavaScript function! eg: () => console.log('gowitness');")
scanCmd.PersistentFlags().StringVar(&opts.Scan.JavaScriptFile, "javascript-file", "", "A file containing a JavaScript function to evaluate on every page, before a screenshot. See --javascript")
+ scanCmd.PersistentFlags().BoolVar(&opts.Scan.SaveContent, "save-content", false, "Save content from network requests to the configured writers. WARNING: This flag has the potential to make your storage explode in size")
// chrome options
scanCmd.PersistentFlags().StringVar(&opts.Chrome.Path, "chrome-path", "", "The path to a Google Chrome binary to use (downloads a platform appropriate binary by default)")
diff --git a/pkg/models/models.go b/pkg/models/models.go
index a7c81bf9..00e8339b 100644
--- a/pkg/models/models.go
+++ b/pkg/models/models.go
@@ -62,7 +62,7 @@ type TLS struct {
KeyExchange string `json:"key_exchange"`
Cipher string `json:"cipher"`
SubjectName string `json:"subject_name"`
- SanList []TLSSanList `json:"san_list"`
+ SanList []TLSSanList `json:"san_list" gorm:"constraint:OnDelete:CASCADE"`
Issuer string `json:"issuer"`
ValidFrom time.Time `json:"valid_from"`
ValidTo time.Time `json:"valid_to"`
@@ -102,6 +102,7 @@ type NetworkLog struct {
RemoteIP string `json:"remote_ip"`
MIMEType string `json:"mime_type"`
Time time.Time `json:"time"`
+ Content []byte `json:"content"`
Error string `json:"error"`
}
diff --git a/pkg/runner/drivers/chromedp.go b/pkg/runner/drivers/chromedp.go
index b04c6015..196c19e6 100644
--- a/pkg/runner/drivers/chromedp.go
+++ b/pkg/runner/drivers/chromedp.go
@@ -12,6 +12,7 @@ import (
"sync"
"time"
+ "github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/cdproto/runtime"
@@ -272,9 +273,30 @@ func (run *Chromedp) Witness(target string, runner *runner.Runner) (*models.Resu
// write the network log
resultMutex.Lock()
+ entryIndex := len(result.Network)
result.Network = append(result.Network, entry)
resultMutex.Unlock()
+ // if we need to write the body, do that
+ // https://github.com/chromedp/chromedp/issues/543
+ if run.options.Scan.SaveContent {
+ go func(index int) {
+ c := chromedp.FromContext(navigationCtx)
+ p := network.GetResponseBody(e.RequestID)
+ body, err := p.Do(cdp.WithExecutor(navigationCtx, c.Target))
+ if err != nil {
+ if run.options.Logging.LogScanErrors {
+ run.log.Error("could not get network request response body", "url", e.Response.URL, "err", err)
+ return
+ }
+ }
+
+ resultMutex.Lock()
+ result.Network[index].Content = body
+ resultMutex.Unlock()
+
+ }(entryIndex)
+ }
}
// mark a request as failed
case *network.EventLoadingFailed:
diff --git a/pkg/runner/drivers/go-rod.go b/pkg/runner/drivers/go-rod.go
index 188907a1..8b81536f 100644
--- a/pkg/runner/drivers/go-rod.go
+++ b/pkg/runner/drivers/go-rod.go
@@ -269,8 +269,26 @@ func (run *Gorod) Witness(target string, runner *runner.Runner) (*models.Result,
// write the network log
resultMutex.Lock()
+ entryIndex := len(result.Network)
result.Network = append(result.Network, entry)
resultMutex.Unlock()
+
+ // if we need to write the body, do that
+ if run.options.Scan.SaveContent {
+ go func(index int) {
+ body, err := proto.NetworkGetResponseBody{RequestID: e.RequestID}.Call(page)
+ if err != nil {
+ if run.options.Logging.LogScanErrors {
+ run.log.Error("could not get network request response body", "url", e.Response.URL, "err", err)
+ return
+ }
+ }
+
+ resultMutex.Lock()
+ result.Network[index].Content = []byte(body.Body)
+ resultMutex.Unlock()
+ }(entryIndex)
+ }
}
return dismissEvents
diff --git a/pkg/runner/options.go b/pkg/runner/options.go
index 4d58ab5b..779a858b 100644
--- a/pkg/runner/options.go
+++ b/pkg/runner/options.go
@@ -76,6 +76,9 @@ type Scan struct {
// JavaScript to evaluate on every page
JavaScript string
JavaScriptFile string
+ // Save content stores content from network requests (warning) this
+ // could make written artefacts huge
+ SaveContent bool
}
// NewDefaultOptions returns Options with some default values
diff --git a/web/ui/src/lib/api/types.ts b/web/ui/src/lib/api/types.ts
index d5159adc..7e16386d 100644
--- a/web/ui/src/lib/api/types.ts
+++ b/web/ui/src/lib/api/types.ts
@@ -96,6 +96,7 @@ interface networklog {
mime_type: string;
time: string;
error: string;
+ content: string;
}
interface consolelog {
diff --git a/web/ui/src/pages/detail/Detail.tsx b/web/ui/src/pages/detail/Detail.tsx
index 07f04721..cb6ba204 100644
--- a/web/ui/src/pages/detail/Detail.tsx
+++ b/web/ui/src/pages/detail/Detail.tsx
@@ -4,7 +4,7 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { ScrollArea } from "@/components/ui/scroll-area";
-import { ExternalLink, ChevronLeft, ChevronRight, Code, ClockIcon, Trash2Icon } from 'lucide-react';
+import { ExternalLink, ChevronLeft, ChevronRight, Code, ClockIcon, Trash2Icon, DownloadIcon } from 'lucide-react';
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, } from "@/components/ui/dialog";
import { WideSkeleton } from '@/components/loading';
import { Form, Link, useParams } from 'react-router-dom';
@@ -38,6 +38,29 @@ const ScreenshotDetailPage = () => {
return "bg-blue-500 text-white";
};
+ const handleDownload = (base64Content: string, url: string) => {
+ const binaryString = atob(base64Content);
+
+ // Create an array of 8-bit unsigned integers from the binary string
+ const binaryLength = binaryString.length;
+ const bytes = new Uint8Array(binaryLength);
+ for (let i = 0; i < binaryLength; i++) {
+ bytes[i] = binaryString.charCodeAt(i);
+ }
+
+ const blob = new Blob([bytes], { type: 'application/octet-stream' });
+ const link = document.createElement('a');
+ link.href = window.URL.createObjectURL(blob);
+
+ // Extract the last part of the URL for the filename
+ const urlParts = new URL(url).pathname.split('/');
+ const fileName = urlParts[urlParts.length - 1] || 'download';
+ link.download = fileName;
+
+ link.click();
+ window.URL.revokeObjectURL(link.href);
+ };
+
if (loading) return ;
if (!detail) return;
@@ -285,6 +308,7 @@ const ScreenshotDetailPage = () => {
HTTP
+
URL
@@ -313,6 +337,16 @@ const ScreenshotDetailPage = () => {
+
+ {log.content && log.content.length > 0 && (
+ handleDownload(log.content, log.url)}
+ >
+
+
+ )}
+
copyToClipboard(log.url, 'URL')}