From aa8478f07217b5a44424b5c102220127f8585a1f Mon Sep 17 00:00:00 2001 From: eltrac Date: Wed, 4 Nov 2026 23:26:12 +0800 Subject: initial commit project: go mod init data, reader: add types and roughly implement Bunny.net reader analytics: basic analyzer, experimental --- analytics/analyzer.go | 71 ++++++++++++++++++++++++++++ analytics/ja4.go | 107 ++++++++++++++++++++++++++++++++++++++++++ analytics/ja4plus-mapping.csv | 67 ++++++++++++++++++++++++++ data/types.go | 23 +++++++++ go.mod | 9 ++++ go.sum | 6 +++ reader/bunny.go | 81 ++++++++++++++++++++++++++++++++ 7 files changed, 364 insertions(+) create mode 100644 analytics/analyzer.go create mode 100644 analytics/ja4.go create mode 100644 analytics/ja4plus-mapping.csv create mode 100644 data/types.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 reader/bunny.go diff --git a/analytics/analyzer.go b/analytics/analyzer.go new file mode 100644 index 0000000..ac8ff03 --- /dev/null +++ b/analytics/analyzer.go @@ -0,0 +1,71 @@ +package analytics + +import ( + "cmp" + "slices" + "strings" + + "github.com/x-way/crawlerdetect" + "src.eltr.ac/eltrac/audidi/data" +) + +type AnalyticsResult struct { + Client string + // 0 = Human + // 1 = Human operating non-browser tools + // 2 = Automated crawler + // 3 = Malicious crawler + BotIndex float32 +} + +func Analyze(log data.LogEntry) *AnalyticsResult { + uaResult := analyzeUA(log.UserAgent) + ja4Client, ja4Result := analyzeJA4(log.JA4) + matchScore := analyzeClientMatch(log.UserAgent, ja4Client) + ASNScore := analyzeASN(log.ASN) + + return &AnalyticsResult{ + Client: cmp.Or(ja4Client, log.UserAgent), + BotIndex: (uaResult + ja4Result + matchScore + ASNScore) / 4, + } +} + +func analyzeUA(ua string) float32 { + if crawlerdetect.IsCrawler(ua) { + return 2 + } else { + return 0 + } +} + +func analyzeJA4(ja4 string) (client string, botIndex float32) { + client, clientType := LookupClientByJA4(ja4) + + return client, float32(clientType) +} + +func analyzeClientMatch(ua string, ja4client string) float32 { + uaClient := strings.ToLower(ua) + ja4Client := strings.ToLower(ja4client) + + if strings.Contains(uaClient, ja4Client) { + return 0 + } else { + return 3 + } +} + +func analyzeASN(asn int) float32 { + DataCenterASNs := []int{ + 16509, 14618, 8987, 38895, 8075, 8068, 12076, 15169, 396982, 19527, 31898, 36351, + 37963, 45102, 24429, 132203, 45090, 55990, 136907, 38365, + 13335, 20940, 16625, 21342, 63949, 54113, 60068, 212238, 15133, 19551, + 14061, 20473, 16276, 24940, 51167, 12876, 60781, 28753, 9009, + 53667, 49505, 9123, 197540, 8560, 6724, 210644, 55293, + 26496, 46606, 32244, 26347, 33070, 19994, 22612, 19871, 2635, + } + if slices.Contains(DataCenterASNs, asn) { + return 2 + } + return 0 +} diff --git a/analytics/ja4.go b/analytics/ja4.go new file mode 100644 index 0000000..56b885c --- /dev/null +++ b/analytics/ja4.go @@ -0,0 +1,107 @@ +package analytics + +import ( + "cmp" + _ "embed" + "log" + "slices" + "strings" + + "github.com/gocarina/gocsv" +) + +//go:embed ja4plus-mapping.csv +var ja4MappingBytes []byte + +type ja4MappingEntry struct { + Application string `csv:"Application"` + Library string `csv:"Library"` + Device string `csv:"Device"` + OS string `csv:"OS"` + Ja4 string `csv:"ja4"` + Ja4s string `csv:"ja4s"` + Ja4h string `csv:"ja4h"` + Ja4x string `csv:"ja4x"` + Ja4t string `csv:"ja4t"` + Ja4tscan string `csv:"ja4tscan"` + Notes string `csv:"Notes"` +} + +type ClientType float32 + +const ( + ClientApp ClientType = 0 + ClientBrowser ClientType = 0 + ClientProxy ClientType = 0.5 + ClientCLI ClientType = 1.5 + ClientOS ClientType = 2 + ClientDevice ClientType = 1.5 + ClientLibrary ClientType = 2 + ClientMalware ClientType = 3 + ClientUnknown ClientType = 2 +) + +func (e ja4MappingEntry) Name() (clientName string, clientType ClientType) { + if app := e.Application; app != "" { + browsers := []string{"Chromium Browser", "Mozilla Firefox", "Safari"} + proxies := []string{"Lumma Client", "SoftEther VPN Client", "SoftEther VPN Server", "ngrok 3.8.0"} + clis := []string{"Curl"} + malwares := []string{ + "Cobalt Strike beacon", + "Cobalt Strike Cat C2", + "Cobalt Strike v4.9.1 beacon", + "Darkgate Client", + "IcedID", + "IcedID Dropper", + "Pikabot C2", + "Qakbot C2", + "Sliver Agent", + "Sliver/Havoc C2 Server", + } + if slices.Contains(browsers, app) { + return app, ClientBrowser + } + if slices.Contains(proxies, app) { + return app, ClientProxy + } + if slices.Contains(clis, app) { + return app, ClientCLI + } + if name := strings.TrimSpace(app); slices.Contains(malwares, name) { + return name, ClientMalware + } + return app, ClientApp + } + + if lib := e.Library; lib != "" { + return lib, ClientLibrary + } + + if dev := e.Device; dev != "" { + return dev, ClientDevice + } + + if os := e.OS; os != "" { + return os, ClientOS + } + + return cmp.Or(e.Application, e.Library, e.Device, e.OS), ClientUnknown +} + +// LookupClient takes a JA4 fingerprint string and +// returns the corresponding HTTP client name string +func LookupClientByJA4(ja4 string) (string, ClientType) { + var mappings []ja4MappingEntry + err := gocsv.UnmarshalBytes(ja4MappingBytes, &mappings) + if err != nil { + log.Fatalln("error unmarshalling JA4 mapping") + } + + for _, entry := range mappings { + if entry.Ja4 == ja4 { + return entry.Name() + } + } + + return "Unknown", ClientUnknown +} diff --git a/analytics/ja4plus-mapping.csv b/analytics/ja4plus-mapping.csv new file mode 100644 index 0000000..870bc3f --- /dev/null +++ b/analytics/ja4plus-mapping.csv @@ -0,0 +1,67 @@ +Application,Library,Device,OS,ja4,ja4s,ja4h,ja4x,ja4t,ja4tscan,Notes +,Python,,,t13i181000_85036bcba153_d41ae481755e,,,,,, +,Python,,,t13d181000_85036bcba153_d41ae481755e,,,,,, +,Python,,,t13d4312h1_c7886603b240_b26ce05bbdd6,,,,,, +Chromium Browser,,,,t13d1516h2_8daaf6152771_02713d6af862,,,,,, +Chromium Browser,,,,q13d0312h3_55b375c5d22e_06cda9e17597,,,,,, +Chromium Browser,,,,t13d1517h2_8daaf6152771_b0da82dd1658,,,,,, +Chromium Browser,,,,t13d1517h2_8daaf6152771_b1ff8ab2d16f,,,,,, +Chromium Browser,,,,t13i1515h2_8daaf6152771_02713d6af862,,,,,, +Chromium Browser,,,,q13i0311h3_55b375c5d22e_06cda9e17597,,,,,, +Chromium Browser,,,,t13i1516h2_8daaf6152771_b0da82dd1658,,,,,, +Chromium Browser,,,,t13i1516h2_8daaf6152771_b1ff8ab2d16f,,,,,, +Mozilla Firefox,,,,t13d1715h2_5b57614c22b0_7121afd63204,,,,,, +Mozilla Firefox,,,,t13i1714h2_5b57614c22b0_7121afd63204,,,,,, +Safari,,,,t13d2014h2_a09f3c656075_14788d8d241b,,,,,, +Safari,,,,t13i2013h2_a09f3c656075_14788d8d241b,,,,,, +,,,Windows 10,,,,,64240_2-1-3-1-1-4_1460_8,, +,,,WSL Ubuntu 22.04,,,,,64240_2-4-8-1-3_1460_7,, +,,,AWS Linux 2,,,,,62727_2-4-8-1-3_8961_7,, +,,,Ubuntu 22.04,,,,,65535_2-4-8-1-3_1460_8,, +,,,Mac OSX/iPhone,,,,,65535_2-1-3-1-1-8-4-0-0_1460_6,, +,GoLang,,,t13d190900_9dc949149365_97f8aa674fd9,,,,,, +,GoLang,,,t13i190800_9dc949149365_97f8aa674fd9,,,,,, +Sliver Agent,GoLang,,,t13d190900_9dc949149365_97f8aa674fd9,t130200_1301_a56c5b993250,,,,, +Sliver Agent,GoLang,,,t13i190800_9dc949149365_97f8aa674fd9,t130200_1301_a56c5b993250,,,,, +Sliver/Havoc C2 Server,,,,,,,000000000000_4f24da86fad6_bf0f0589fc03,,, +Sliver/Havoc C2 Server,,,,,,,000000000000_7c32fa18c13e_bf0f0589fc03,,, +IcedID Dropper,,,,,,ge11cn020000_9ed1ff1f7b03_cd8dafe26982,,,, +IcedID ,,,,t13d201100_2b729b4bf6f3_9e7b989ebec8,t120300_c030_5e2616a54c73,,,,, +Cobalt Strike Cat C2,,,,,,,2166164053c1_2166164053c1_30d204a01551,,, +Cobalt Strike beacon,,,,,,ge11cn060000_4e59edc1297a_4da5efaf0cb,,,, +SoftEther VPN Client,,,,t13d880900_fcb5b95cb75a_b0d3b4ac2a14,,,,,, +SoftEther VPN Client,,,,t13i880900_fcb5b95cb75a_b0d3b4ac2a14,,,,,, +SoftEther VPN Server,,,,,,,d55f458d5a6c_d55f458d5a6c_0fc8c171b6ae,,, +Qakbot C2,,,,,,,2bab15409345_af684594efb4_000000000000,,, +Pikabot C2,,,,,,,1a59268f55e5_1a59268f55e5_795797892f9c,,,NOT Aegir +Darkgate Client,,,,,,po10nn060000_cdb958d032b0,,,, +Lumma Client,,,,,,po11nn050000_d253db9d024b,,,, +,,Epson Printer,,,,,,,28960_2-4-8-1-3_1460_3_1-4-8-16, +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12i190700_d83cc789557e_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over wininet +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12i210700_76e208dd3e22_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over winhttp +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12d190800_d83cc789557e_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over wininet +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12d210800_76e208dd3e22_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over winhttp +,,Ubiquiti Router,UniFi OS 3.2.12,,,,,,43440_2-4-8-1-3_1460_12_1-2-4-8-17, +,,Ubiquiti Camera,,,,,,,28960_2-4-8-1-3_1460_6_1-2-4-8-16, +,,Rachio IoT,,,,,,,7168_2-3-0_1460_0_0-0-0-0-0-0-0-0-0-0, +,,Phillips Hue Bridge,,,,,,,29200_2-1-1-4-1-3_1460_3_1-2-4-8-17, +,GoLang net package,,,t13d191000_9dc949149365_e7c285222651,,,,,, +,GoLang webhooks,,,t13d1412h2_e33ad33b3d25_6b314db333b6,,,,,, +,GoLang,,,t12d160700_8cdfa2d4673b_18dd7303c4a5,,,,,, +,GoLang,,,t13d141000_cbb2034c60b8_e7c285222651,,,,,, +Chromium Browser,,,,,,ge11nn08enus_050dd5cfb971_000000000000_000000000000,,,,Connection to new site over http 1.1 +Chromium Browser,,,,,,ge11nr07enus_a3e0c85f2f9f_000000000000_000000000000,,,,http 1.1 +Chromium Browser,,,,,,ge11nr08enus_307986a18ea3_000000000000_000000000000,,,,http 1.1 +ngrok 3.8.0,,,,,,ge11nn17enus_d488a5d46f42_000000000000_000000000000,,,,Chrome through ngrok +ngrok 3.8.0,,,,,,ge11nr15enus_a978addab902_000000000000_000000000000,,,,Chrome through ngrok +,,Cisco RV32x,,,,,fba10053814e_fba10053814e_795797892f9c,,5792_2-4-8-1-3_1460_4_3-6-12-24, +Curl,,,,,,ge11nn030000_fe444ad14866_000000000000_000000000000,,,, +Telegram,,,,,,po11nn07enus_b9cd978c9817_000000000000_000000000000,,,, +,,LIFX Smart Bulbs,,t12d520600_b380db6257eb_0a9c83bf8b96,,,,5744_2_1436_00,, +,,Nest,,t12d8008h1_9cedc1f1428b_046e095b7c4a,,,,29200_2-4-8-1-3_1460_6,, +,,Phillips Hue Bridge,,t12d350600_9d4c96c0953b_0a9c83bf8b96,,,,29200_2-1-1-4-1-3_1460_3,, +,WinINET,,Windows 10/11,t12d190800_d83cc789557e_7af1ed941c26,,,,,, +,GoLang,,,t12d190800_d83cc789557e_7af1ed941c26,,,,,, +,,Cloudflare,,,,,,,65535_2-4-8-1-3_1400_13_1, +,,Cloudflare,,,,,,65535_2-4-8-1-3_1400_13,,JA4TS when connecting to Cloudflare from Mac/Unix +,,Cloudflare,,,,,,65535_2-1-1-4-1-3_1400_13,,JA4TS when connecting to Cloudflare from Windows diff --git a/data/types.go b/data/types.go new file mode 100644 index 0000000..3becae8 --- /dev/null +++ b/data/types.go @@ -0,0 +1,23 @@ +package data + +import "io" + +type LogEntry struct { + // requeted resource + Host string + Path string + URL string + Status int + // client network + CountryCode string + IP string + ASN int + // client info + UserAgent string + JA4 string + JA3 string +} + +type LogReader interface { + Read(input io.Reader) ([]LogEntry, error) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7507e58 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module src.eltr.ac/eltrac/audidi + +go 1.26.5 + +require ( + github.com/gocarina/gocsv v0.0.0-20261101205459-d0dd027dcbfa // indirect + github.com/ohler55/ojg v1.28.9 // indirect + github.com/x-way/crawlerdetect v0.2.32 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b882144 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/gocarina/gocsv v0.0.0-20261101205459-d0dd027dcbfa h1:u6dQLh7NdeVIBn4fE4YL9PiHFephNp7t7rUSU+fulro= +github.com/gocarina/gocsv v0.0.0-20261101205459-d0dd027dcbfa/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/ohler55/ojg v1.28.9 h1:VL2vdN1CjniaNABNOFjorAW+/o7eBQDc4z5jO49jkHM= +github.com/ohler55/ojg v1.28.9/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= +github.com/x-way/crawlerdetect v0.2.32 h1:kmYHC8lk5T91PDETQ3h9pF35GUyrdzmi3KX64jPvHt4= +github.com/x-way/crawlerdetect v0.2.32/go.mod h1:BPHLsB3FOuiwoWyhAvnqeiUSAEKd34O7BcsTCcxHRj4= diff --git a/reader/bunny.go b/reader/bunny.go new file mode 100644 index 0000000..4a94913 --- /dev/null +++ b/reader/bunny.go @@ -0,0 +1,81 @@ +package reader + +import ( + "fmt" + "io" + + "github.com/ohler55/ojg/oj" + + "src.eltr.ac/eltrac/audidi/data" +) + +type bunnyJSON struct { + Timestamp string `json:"timestamp"` + PullZoneID int `json:"pullZoneId"` + RequestID string `json:"requestId"` + CacheStatus string `json:"cacheStatus"` + StatusCode int `json:"statusCode"` + BytesSent int `json:"bytesSent"` + RemoteIP string `json:"remoteIp"` + CountryCode string `json:"countryCode"` + EdgeLocation string `json:"edgeLocation"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Path string `json:"path"` + URL string `json:"url"` + UserAgent string `json:"userAgent"` + Referer string `json:"referer"` + JA4Fingerprint string `json:"ja4Fingerprint"` + ASN int `json:"asn"` + ASNOrganization string `json:"asnOrganization"` +} + +type bunnyError struct { + Code string `json:"code"` + Message string `json:"message"` + Details []string `json:"details"` +} + +type bunnyResponse struct { + Data []bunnyJSON `json:"data"` + Error *bunnyError `json:"error"` +} + +// BunnyReader reads log entries from the Bunny.net Logging API v2. +type BunnyReader struct{} + +// Read parses Bunny.net log records from input. +func (r *BunnyReader) Read(input io.Reader) ([]data.LogEntry, error) { + b, err := io.ReadAll(input) + if err != nil { + return nil, fmt.Errorf("error reading from input") + } + + var parser oj.Parser + var response bunnyResponse + parser.Unmarshal(b, &response) + + if err := response.Error; err != nil { + return nil, fmt.Errorf("API response error %s: %s", err.Code, err.Message) + } + + var result []data.LogEntry + if logdata := response.Data; len(logdata) > 0 { + for _, entry := range logdata { + result = append(result, data.LogEntry{ + Host: entry.Host, + Path: entry.Path, + URL: entry.URL, + Status: entry.StatusCode, + CountryCode: entry.CountryCode, + IP: entry.RemoteIP, + ASN: entry.ASN, + UserAgent: entry.UserAgent, + JA4: entry.JA4Fingerprint, + JA3: "", + }) + } + } + + return result, nil +} -- cgit v1.2.3