summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--analytics/analyzer.go71
-rw-r--r--analytics/ja4.go107
-rw-r--r--analytics/ja4plus-mapping.csv67
-rw-r--r--data/types.go23
-rw-r--r--go.mod9
-rw-r--r--go.sum6
-rw-r--r--reader/bunny.go81
7 files changed, 364 insertions, 0 deletions
diff --git a/analytics/analyzer.go b/analytics/analyzer.go
new file mode 100644
index 0000000..ac8ff03
--- /dev/null
+++ b/analytics/analyzer.go
@@ -0,0 +1,71 @@
+package analytics
+
+import (
+ "cmp"
+ "slices"
+ "strings"
+
+ "github.com/x-way/crawlerdetect"
+ "src.eltr.ac/eltrac/audidi/data"
+)
+
+type AnalyticsResult struct {
+ Client string
+ // 0 = Human
+ // 1 = Human operating non-browser tools
+ // 2 = Automated crawler
+ // 3 = Malicious crawler
+ BotIndex float32
+}
+
+func Analyze(log data.LogEntry) *AnalyticsResult {
+ uaResult := analyzeUA(log.UserAgent)
+ ja4Client, ja4Result := analyzeJA4(log.JA4)
+ matchScore := analyzeClientMatch(log.UserAgent, ja4Client)
+ ASNScore := analyzeASN(log.ASN)
+
+ return &AnalyticsResult{
+ Client: cmp.Or(ja4Client, log.UserAgent),
+ BotIndex: (uaResult + ja4Result + matchScore + ASNScore) / 4,
+ }
+}
+
+func analyzeUA(ua string) float32 {
+ if crawlerdetect.IsCrawler(ua) {
+ return 2
+ } else {
+ return 0
+ }
+}
+
+func analyzeJA4(ja4 string) (client string, botIndex float32) {
+ client, clientType := LookupClientByJA4(ja4)
+
+ return client, float32(clientType)
+}
+
+func analyzeClientMatch(ua string, ja4client string) float32 {
+ uaClient := strings.ToLower(ua)
+ ja4Client := strings.ToLower(ja4client)
+
+ if strings.Contains(uaClient, ja4Client) {
+ return 0
+ } else {
+ return 3
+ }
+}
+
+func analyzeASN(asn int) float32 {
+ DataCenterASNs := []int{
+ 16509, 14618, 8987, 38895, 8075, 8068, 12076, 15169, 396982, 19527, 31898, 36351,
+ 37963, 45102, 24429, 132203, 45090, 55990, 136907, 38365,
+ 13335, 20940, 16625, 21342, 63949, 54113, 60068, 212238, 15133, 19551,
+ 14061, 20473, 16276, 24940, 51167, 12876, 60781, 28753, 9009,
+ 53667, 49505, 9123, 197540, 8560, 6724, 210644, 55293,
+ 26496, 46606, 32244, 26347, 33070, 19994, 22612, 19871, 2635,
+ }
+ if slices.Contains(DataCenterASNs, asn) {
+ return 2
+ }
+ return 0
+}
diff --git a/analytics/ja4.go b/analytics/ja4.go
new file mode 100644
index 0000000..56b885c
--- /dev/null
+++ b/analytics/ja4.go
@@ -0,0 +1,107 @@
+package analytics
+
+import (
+ "cmp"
+ _ "embed"
+ "log"
+ "slices"
+ "strings"
+
+ "github.com/gocarina/gocsv"
+)
+
+//go:embed ja4plus-mapping.csv
+var ja4MappingBytes []byte
+
+type ja4MappingEntry struct {
+ Application string `csv:"Application"`
+ Library string `csv:"Library"`
+ Device string `csv:"Device"`
+ OS string `csv:"OS"`
+ Ja4 string `csv:"ja4"`
+ Ja4s string `csv:"ja4s"`
+ Ja4h string `csv:"ja4h"`
+ Ja4x string `csv:"ja4x"`
+ Ja4t string `csv:"ja4t"`
+ Ja4tscan string `csv:"ja4tscan"`
+ Notes string `csv:"Notes"`
+}
+
+type ClientType float32
+
+const (
+ ClientApp ClientType = 0
+ ClientBrowser ClientType = 0
+ ClientProxy ClientType = 0.5
+ ClientCLI ClientType = 1.5
+ ClientOS ClientType = 2
+ ClientDevice ClientType = 1.5
+ ClientLibrary ClientType = 2
+ ClientMalware ClientType = 3
+ ClientUnknown ClientType = 2
+)
+
+func (e ja4MappingEntry) Name() (clientName string, clientType ClientType) {
+ if app := e.Application; app != "" {
+ browsers := []string{"Chromium Browser", "Mozilla Firefox", "Safari"}
+ proxies := []string{"Lumma Client", "SoftEther VPN Client", "SoftEther VPN Server", "ngrok 3.8.0"}
+ clis := []string{"Curl"}
+ malwares := []string{
+ "Cobalt Strike beacon",
+ "Cobalt Strike Cat C2",
+ "Cobalt Strike v4.9.1 beacon",
+ "Darkgate Client",
+ "IcedID",
+ "IcedID Dropper",
+ "Pikabot C2",
+ "Qakbot C2",
+ "Sliver Agent",
+ "Sliver/Havoc C2 Server",
+ }
+ if slices.Contains(browsers, app) {
+ return app, ClientBrowser
+ }
+ if slices.Contains(proxies, app) {
+ return app, ClientProxy
+ }
+ if slices.Contains(clis, app) {
+ return app, ClientCLI
+ }
+ if name := strings.TrimSpace(app); slices.Contains(malwares, name) {
+ return name, ClientMalware
+ }
+ return app, ClientApp
+ }
+
+ if lib := e.Library; lib != "" {
+ return lib, ClientLibrary
+ }
+
+ if dev := e.Device; dev != "" {
+ return dev, ClientDevice
+ }
+
+ if os := e.OS; os != "" {
+ return os, ClientOS
+ }
+
+ return cmp.Or(e.Application, e.Library, e.Device, e.OS), ClientUnknown
+}
+
+// LookupClient takes a JA4 fingerprint string and
+// returns the corresponding HTTP client name string
+func LookupClientByJA4(ja4 string) (string, ClientType) {
+ var mappings []ja4MappingEntry
+ err := gocsv.UnmarshalBytes(ja4MappingBytes, &mappings)
+ if err != nil {
+ log.Fatalln("error unmarshalling JA4 mapping")
+ }
+
+ for _, entry := range mappings {
+ if entry.Ja4 == ja4 {
+ return entry.Name()
+ }
+ }
+
+ return "Unknown", ClientUnknown
+}
diff --git a/analytics/ja4plus-mapping.csv b/analytics/ja4plus-mapping.csv
new file mode 100644
index 0000000..870bc3f
--- /dev/null
+++ b/analytics/ja4plus-mapping.csv
@@ -0,0 +1,67 @@
+Application,Library,Device,OS,ja4,ja4s,ja4h,ja4x,ja4t,ja4tscan,Notes
+,Python,,,t13i181000_85036bcba153_d41ae481755e,,,,,,
+,Python,,,t13d181000_85036bcba153_d41ae481755e,,,,,,
+,Python,,,t13d4312h1_c7886603b240_b26ce05bbdd6,,,,,,
+Chromium Browser,,,,t13d1516h2_8daaf6152771_02713d6af862,,,,,,
+Chromium Browser,,,,q13d0312h3_55b375c5d22e_06cda9e17597,,,,,,
+Chromium Browser,,,,t13d1517h2_8daaf6152771_b0da82dd1658,,,,,,
+Chromium Browser,,,,t13d1517h2_8daaf6152771_b1ff8ab2d16f,,,,,,
+Chromium Browser,,,,t13i1515h2_8daaf6152771_02713d6af862,,,,,,
+Chromium Browser,,,,q13i0311h3_55b375c5d22e_06cda9e17597,,,,,,
+Chromium Browser,,,,t13i1516h2_8daaf6152771_b0da82dd1658,,,,,,
+Chromium Browser,,,,t13i1516h2_8daaf6152771_b1ff8ab2d16f,,,,,,
+Mozilla Firefox,,,,t13d1715h2_5b57614c22b0_7121afd63204,,,,,,
+Mozilla Firefox,,,,t13i1714h2_5b57614c22b0_7121afd63204,,,,,,
+Safari,,,,t13d2014h2_a09f3c656075_14788d8d241b,,,,,,
+Safari,,,,t13i2013h2_a09f3c656075_14788d8d241b,,,,,,
+,,,Windows 10,,,,,64240_2-1-3-1-1-4_1460_8,,
+,,,WSL Ubuntu 22.04,,,,,64240_2-4-8-1-3_1460_7,,
+,,,AWS Linux 2,,,,,62727_2-4-8-1-3_8961_7,,
+,,,Ubuntu 22.04,,,,,65535_2-4-8-1-3_1460_8,,
+,,,Mac OSX/iPhone,,,,,65535_2-1-3-1-1-8-4-0-0_1460_6,,
+,GoLang,,,t13d190900_9dc949149365_97f8aa674fd9,,,,,,
+,GoLang,,,t13i190800_9dc949149365_97f8aa674fd9,,,,,,
+Sliver Agent,GoLang,,,t13d190900_9dc949149365_97f8aa674fd9,t130200_1301_a56c5b993250,,,,,
+Sliver Agent,GoLang,,,t13i190800_9dc949149365_97f8aa674fd9,t130200_1301_a56c5b993250,,,,,
+Sliver/Havoc C2 Server,,,,,,,000000000000_4f24da86fad6_bf0f0589fc03,,,
+Sliver/Havoc C2 Server,,,,,,,000000000000_7c32fa18c13e_bf0f0589fc03,,,
+IcedID Dropper,,,,,,ge11cn020000_9ed1ff1f7b03_cd8dafe26982,,,,
+IcedID ,,,,t13d201100_2b729b4bf6f3_9e7b989ebec8,t120300_c030_5e2616a54c73,,,,,
+Cobalt Strike Cat C2,,,,,,,2166164053c1_2166164053c1_30d204a01551,,,
+Cobalt Strike beacon,,,,,,ge11cn060000_4e59edc1297a_4da5efaf0cb,,,,
+SoftEther VPN Client,,,,t13d880900_fcb5b95cb75a_b0d3b4ac2a14,,,,,,
+SoftEther VPN Client,,,,t13i880900_fcb5b95cb75a_b0d3b4ac2a14,,,,,,
+SoftEther VPN Server,,,,,,,d55f458d5a6c_d55f458d5a6c_0fc8c171b6ae,,,
+Qakbot C2,,,,,,,2bab15409345_af684594efb4_000000000000,,,
+Pikabot C2,,,,,,,1a59268f55e5_1a59268f55e5_795797892f9c,,,NOT Aegir
+Darkgate Client,,,,,,po10nn060000_cdb958d032b0,,,,
+Lumma Client,,,,,,po11nn050000_d253db9d024b,,,,
+,,Epson Printer,,,,,,,28960_2-4-8-1-3_1460_3_1-4-8-16,
+Cobalt Strike v4.9.1 beacon,,,Windows 10,t12i190700_d83cc789557e_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over wininet
+Cobalt Strike v4.9.1 beacon,,,Windows 10,t12i210700_76e208dd3e22_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over winhttp
+Cobalt Strike v4.9.1 beacon,,,Windows 10,t12d190800_d83cc789557e_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over wininet
+Cobalt Strike v4.9.1 beacon,,,Windows 10,t12d210800_76e208dd3e22_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over winhttp
+,,Ubiquiti Router,UniFi OS 3.2.12,,,,,,43440_2-4-8-1-3_1460_12_1-2-4-8-17,
+,,Ubiquiti Camera,,,,,,,28960_2-4-8-1-3_1460_6_1-2-4-8-16,
+,,Rachio IoT,,,,,,,7168_2-3-0_1460_0_0-0-0-0-0-0-0-0-0-0,
+,,Phillips Hue Bridge,,,,,,,29200_2-1-1-4-1-3_1460_3_1-2-4-8-17,
+,GoLang net package,,,t13d191000_9dc949149365_e7c285222651,,,,,,
+,GoLang webhooks,,,t13d1412h2_e33ad33b3d25_6b314db333b6,,,,,,
+,GoLang,,,t12d160700_8cdfa2d4673b_18dd7303c4a5,,,,,,
+,GoLang,,,t13d141000_cbb2034c60b8_e7c285222651,,,,,,
+Chromium Browser,,,,,,ge11nn08enus_050dd5cfb971_000000000000_000000000000,,,,Connection to new site over http 1.1
+Chromium Browser,,,,,,ge11nr07enus_a3e0c85f2f9f_000000000000_000000000000,,,,http 1.1
+Chromium Browser,,,,,,ge11nr08enus_307986a18ea3_000000000000_000000000000,,,,http 1.1
+ngrok 3.8.0,,,,,,ge11nn17enus_d488a5d46f42_000000000000_000000000000,,,,Chrome through ngrok
+ngrok 3.8.0,,,,,,ge11nr15enus_a978addab902_000000000000_000000000000,,,,Chrome through ngrok
+,,Cisco RV32x,,,,,fba10053814e_fba10053814e_795797892f9c,,5792_2-4-8-1-3_1460_4_3-6-12-24,
+Curl,,,,,,ge11nn030000_fe444ad14866_000000000000_000000000000,,,,
+Telegram,,,,,,po11nn07enus_b9cd978c9817_000000000000_000000000000,,,,
+,,LIFX Smart Bulbs,,t12d520600_b380db6257eb_0a9c83bf8b96,,,,5744_2_1436_00,,
+,,Nest,,t12d8008h1_9cedc1f1428b_046e095b7c4a,,,,29200_2-4-8-1-3_1460_6,,
+,,Phillips Hue Bridge,,t12d350600_9d4c96c0953b_0a9c83bf8b96,,,,29200_2-1-1-4-1-3_1460_3,,
+,WinINET,,Windows 10/11,t12d190800_d83cc789557e_7af1ed941c26,,,,,,
+,GoLang,,,t12d190800_d83cc789557e_7af1ed941c26,,,,,,
+,,Cloudflare,,,,,,,65535_2-4-8-1-3_1400_13_1,
+,,Cloudflare,,,,,,65535_2-4-8-1-3_1400_13,,JA4TS when connecting to Cloudflare from Mac/Unix
+,,Cloudflare,,,,,,65535_2-1-1-4-1-3_1400_13,,JA4TS when connecting to Cloudflare from Windows
diff --git a/data/types.go b/data/types.go
new file mode 100644
index 0000000..3becae8
--- /dev/null
+++ b/data/types.go
@@ -0,0 +1,23 @@
+package data
+
+import "io"
+
+type LogEntry struct {
+ // requeted resource
+ Host string
+ Path string
+ URL string
+ Status int
+ // client network
+ CountryCode string
+ IP string
+ ASN int
+ // client info
+ UserAgent string
+ JA4 string
+ JA3 string
+}
+
+type LogReader interface {
+ Read(input io.Reader) ([]LogEntry, error)
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..7507e58
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,9 @@
+module src.eltr.ac/eltrac/audidi
+
+go 1.26.5
+
+require (
+ github.com/gocarina/gocsv v0.0.0-20261101205459-d0dd027dcbfa // indirect
+ github.com/ohler55/ojg v1.28.9 // indirect
+ github.com/x-way/crawlerdetect v0.2.32 // indirect
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..b882144
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,6 @@
+github.com/gocarina/gocsv v0.0.0-20261101205459-d0dd027dcbfa h1:u6dQLh7NdeVIBn4fE4YL9PiHFephNp7t7rUSU+fulro=
+github.com/gocarina/gocsv v0.0.0-20261101205459-d0dd027dcbfa/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
+github.com/ohler55/ojg v1.28.9 h1:VL2vdN1CjniaNABNOFjorAW+/o7eBQDc4z5jO49jkHM=
+github.com/ohler55/ojg v1.28.9/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o=
+github.com/x-way/crawlerdetect v0.2.32 h1:kmYHC8lk5T91PDETQ3h9pF35GUyrdzmi3KX64jPvHt4=
+github.com/x-way/crawlerdetect v0.2.32/go.mod h1:BPHLsB3FOuiwoWyhAvnqeiUSAEKd34O7BcsTCcxHRj4=
diff --git a/reader/bunny.go b/reader/bunny.go
new file mode 100644
index 0000000..4a94913
--- /dev/null
+++ b/reader/bunny.go
@@ -0,0 +1,81 @@
+package reader
+
+import (
+ "fmt"
+ "io"
+
+ "github.com/ohler55/ojg/oj"
+
+ "src.eltr.ac/eltrac/audidi/data"
+)
+
+type bunnyJSON struct {
+ Timestamp string `json:"timestamp"`
+ PullZoneID int `json:"pullZoneId"`
+ RequestID string `json:"requestId"`
+ CacheStatus string `json:"cacheStatus"`
+ StatusCode int `json:"statusCode"`
+ BytesSent int `json:"bytesSent"`
+ RemoteIP string `json:"remoteIp"`
+ CountryCode string `json:"countryCode"`
+ EdgeLocation string `json:"edgeLocation"`
+ Scheme string `json:"scheme"`
+ Host string `json:"host"`
+ Path string `json:"path"`
+ URL string `json:"url"`
+ UserAgent string `json:"userAgent"`
+ Referer string `json:"referer"`
+ JA4Fingerprint string `json:"ja4Fingerprint"`
+ ASN int `json:"asn"`
+ ASNOrganization string `json:"asnOrganization"`
+}
+
+type bunnyError struct {
+ Code string `json:"code"`
+ Message string `json:"message"`
+ Details []string `json:"details"`
+}
+
+type bunnyResponse struct {
+ Data []bunnyJSON `json:"data"`
+ Error *bunnyError `json:"error"`
+}
+
+// BunnyReader reads log entries from the Bunny.net Logging API v2.
+type BunnyReader struct{}
+
+// Read parses Bunny.net log records from input.
+func (r *BunnyReader) Read(input io.Reader) ([]data.LogEntry, error) {
+ b, err := io.ReadAll(input)
+ if err != nil {
+ return nil, fmt.Errorf("error reading from input")
+ }
+
+ var parser oj.Parser
+ var response bunnyResponse
+ parser.Unmarshal(b, &response)
+
+ if err := response.Error; err != nil {
+ return nil, fmt.Errorf("API response error %s: %s", err.Code, err.Message)
+ }
+
+ var result []data.LogEntry
+ if logdata := response.Data; len(logdata) > 0 {
+ for _, entry := range logdata {
+ result = append(result, data.LogEntry{
+ Host: entry.Host,
+ Path: entry.Path,
+ URL: entry.URL,
+ Status: entry.StatusCode,
+ CountryCode: entry.CountryCode,
+ IP: entry.RemoteIP,
+ ASN: entry.ASN,
+ UserAgent: entry.UserAgent,
+ JA4: entry.JA4Fingerprint,
+ JA3: "",
+ })
+ }
+ }
+
+ return result, nil
+}