diff options
Diffstat (limited to 'analytics')
| -rw-r--r-- | analytics/analyzer.go | 71 | ||||
| -rw-r--r-- | analytics/ja4.go | 107 | ||||
| -rw-r--r-- | analytics/ja4plus-mapping.csv | 67 |
3 files changed, 245 insertions, 0 deletions
diff --git a/analytics/analyzer.go b/analytics/analyzer.go new file mode 100644 index 0000000..ac8ff03 --- /dev/null +++ b/analytics/analyzer.go @@ -0,0 +1,71 @@ +package analytics + +import ( + "cmp" + "slices" + "strings" + + "github.com/x-way/crawlerdetect" + "src.eltr.ac/eltrac/audidi/data" +) + +type AnalyticsResult struct { + Client string + // 0 = Human + // 1 = Human operating non-browser tools + // 2 = Automated crawler + // 3 = Malicious crawler + BotIndex float32 +} + +func Analyze(log data.LogEntry) *AnalyticsResult { + uaResult := analyzeUA(log.UserAgent) + ja4Client, ja4Result := analyzeJA4(log.JA4) + matchScore := analyzeClientMatch(log.UserAgent, ja4Client) + ASNScore := analyzeASN(log.ASN) + + return &AnalyticsResult{ + Client: cmp.Or(ja4Client, log.UserAgent), + BotIndex: (uaResult + ja4Result + matchScore + ASNScore) / 4, + } +} + +func analyzeUA(ua string) float32 { + if crawlerdetect.IsCrawler(ua) { + return 2 + } else { + return 0 + } +} + +func analyzeJA4(ja4 string) (client string, botIndex float32) { + client, clientType := LookupClientByJA4(ja4) + + return client, float32(clientType) +} + +func analyzeClientMatch(ua string, ja4client string) float32 { + uaClient := strings.ToLower(ua) + ja4Client := strings.ToLower(ja4client) + + if strings.Contains(uaClient, ja4Client) { + return 0 + } else { + return 3 + } +} + +func analyzeASN(asn int) float32 { + DataCenterASNs := []int{ + 16509, 14618, 8987, 38895, 8075, 8068, 12076, 15169, 396982, 19527, 31898, 36351, + 37963, 45102, 24429, 132203, 45090, 55990, 136907, 38365, + 13335, 20940, 16625, 21342, 63949, 54113, 60068, 212238, 15133, 19551, + 14061, 20473, 16276, 24940, 51167, 12876, 60781, 28753, 9009, + 53667, 49505, 9123, 197540, 8560, 6724, 210644, 55293, + 26496, 46606, 32244, 26347, 33070, 19994, 22612, 19871, 2635, + } + if slices.Contains(DataCenterASNs, asn) { + return 2 + } + return 0 +} diff --git a/analytics/ja4.go b/analytics/ja4.go new file mode 100644 index 0000000..56b885c --- /dev/null +++ b/analytics/ja4.go @@ -0,0 +1,107 @@ +package analytics
+
+import (
+ "cmp"
+ _ "embed"
+ "log"
+ "slices"
+ "strings"
+
+ "github.com/gocarina/gocsv"
+)
+
+//go:embed ja4plus-mapping.csv
+var ja4MappingBytes []byte
+
+type ja4MappingEntry struct {
+ Application string `csv:"Application"`
+ Library string `csv:"Library"`
+ Device string `csv:"Device"`
+ OS string `csv:"OS"`
+ Ja4 string `csv:"ja4"`
+ Ja4s string `csv:"ja4s"`
+ Ja4h string `csv:"ja4h"`
+ Ja4x string `csv:"ja4x"`
+ Ja4t string `csv:"ja4t"`
+ Ja4tscan string `csv:"ja4tscan"`
+ Notes string `csv:"Notes"`
+}
+
+type ClientType float32
+
+const (
+ ClientApp ClientType = 0
+ ClientBrowser ClientType = 0
+ ClientProxy ClientType = 0.5
+ ClientCLI ClientType = 1.5
+ ClientOS ClientType = 2
+ ClientDevice ClientType = 1.5
+ ClientLibrary ClientType = 2
+ ClientMalware ClientType = 3
+ ClientUnknown ClientType = 2
+)
+
+func (e ja4MappingEntry) Name() (clientName string, clientType ClientType) {
+ if app := e.Application; app != "" {
+ browsers := []string{"Chromium Browser", "Mozilla Firefox", "Safari"}
+ proxies := []string{"Lumma Client", "SoftEther VPN Client", "SoftEther VPN Server", "ngrok 3.8.0"}
+ clis := []string{"Curl"}
+ malwares := []string{
+ "Cobalt Strike beacon",
+ "Cobalt Strike Cat C2",
+ "Cobalt Strike v4.9.1 beacon",
+ "Darkgate Client",
+ "IcedID",
+ "IcedID Dropper",
+ "Pikabot C2",
+ "Qakbot C2",
+ "Sliver Agent",
+ "Sliver/Havoc C2 Server",
+ }
+ if slices.Contains(browsers, app) {
+ return app, ClientBrowser
+ }
+ if slices.Contains(proxies, app) {
+ return app, ClientProxy
+ }
+ if slices.Contains(clis, app) {
+ return app, ClientCLI
+ }
+ if name := strings.TrimSpace(app); slices.Contains(malwares, name) {
+ return name, ClientMalware
+ }
+ return app, ClientApp
+ }
+
+ if lib := e.Library; lib != "" {
+ return lib, ClientLibrary
+ }
+
+ if dev := e.Device; dev != "" {
+ return dev, ClientDevice
+ }
+
+ if os := e.OS; os != "" {
+ return os, ClientOS
+ }
+
+ return cmp.Or(e.Application, e.Library, e.Device, e.OS), ClientUnknown
+}
+
+// LookupClient takes a JA4 fingerprint string and
+// returns the corresponding HTTP client name string
+func LookupClientByJA4(ja4 string) (string, ClientType) {
+ var mappings []ja4MappingEntry
+ err := gocsv.UnmarshalBytes(ja4MappingBytes, &mappings)
+ if err != nil {
+ log.Fatalln("error unmarshalling JA4 mapping")
+ }
+
+ for _, entry := range mappings {
+ if entry.Ja4 == ja4 {
+ return entry.Name()
+ }
+ }
+
+ return "Unknown", ClientUnknown
+}
diff --git a/analytics/ja4plus-mapping.csv b/analytics/ja4plus-mapping.csv new file mode 100644 index 0000000..870bc3f --- /dev/null +++ b/analytics/ja4plus-mapping.csv @@ -0,0 +1,67 @@ +Application,Library,Device,OS,ja4,ja4s,ja4h,ja4x,ja4t,ja4tscan,Notes +,Python,,,t13i181000_85036bcba153_d41ae481755e,,,,,, +,Python,,,t13d181000_85036bcba153_d41ae481755e,,,,,, +,Python,,,t13d4312h1_c7886603b240_b26ce05bbdd6,,,,,, +Chromium Browser,,,,t13d1516h2_8daaf6152771_02713d6af862,,,,,, +Chromium Browser,,,,q13d0312h3_55b375c5d22e_06cda9e17597,,,,,, +Chromium Browser,,,,t13d1517h2_8daaf6152771_b0da82dd1658,,,,,, +Chromium Browser,,,,t13d1517h2_8daaf6152771_b1ff8ab2d16f,,,,,, +Chromium Browser,,,,t13i1515h2_8daaf6152771_02713d6af862,,,,,, +Chromium Browser,,,,q13i0311h3_55b375c5d22e_06cda9e17597,,,,,, +Chromium Browser,,,,t13i1516h2_8daaf6152771_b0da82dd1658,,,,,, +Chromium Browser,,,,t13i1516h2_8daaf6152771_b1ff8ab2d16f,,,,,, +Mozilla Firefox,,,,t13d1715h2_5b57614c22b0_7121afd63204,,,,,, +Mozilla Firefox,,,,t13i1714h2_5b57614c22b0_7121afd63204,,,,,, +Safari,,,,t13d2014h2_a09f3c656075_14788d8d241b,,,,,, +Safari,,,,t13i2013h2_a09f3c656075_14788d8d241b,,,,,, +,,,Windows 10,,,,,64240_2-1-3-1-1-4_1460_8,, +,,,WSL Ubuntu 22.04,,,,,64240_2-4-8-1-3_1460_7,, +,,,AWS Linux 2,,,,,62727_2-4-8-1-3_8961_7,, +,,,Ubuntu 22.04,,,,,65535_2-4-8-1-3_1460_8,, +,,,Mac OSX/iPhone,,,,,65535_2-1-3-1-1-8-4-0-0_1460_6,, +,GoLang,,,t13d190900_9dc949149365_97f8aa674fd9,,,,,, +,GoLang,,,t13i190800_9dc949149365_97f8aa674fd9,,,,,, +Sliver Agent,GoLang,,,t13d190900_9dc949149365_97f8aa674fd9,t130200_1301_a56c5b993250,,,,, +Sliver Agent,GoLang,,,t13i190800_9dc949149365_97f8aa674fd9,t130200_1301_a56c5b993250,,,,, +Sliver/Havoc C2 Server,,,,,,,000000000000_4f24da86fad6_bf0f0589fc03,,, +Sliver/Havoc C2 Server,,,,,,,000000000000_7c32fa18c13e_bf0f0589fc03,,, +IcedID Dropper,,,,,,ge11cn020000_9ed1ff1f7b03_cd8dafe26982,,,, +IcedID ,,,,t13d201100_2b729b4bf6f3_9e7b989ebec8,t120300_c030_5e2616a54c73,,,,, +Cobalt Strike Cat C2,,,,,,,2166164053c1_2166164053c1_30d204a01551,,, +Cobalt Strike beacon,,,,,,ge11cn060000_4e59edc1297a_4da5efaf0cb,,,, +SoftEther VPN Client,,,,t13d880900_fcb5b95cb75a_b0d3b4ac2a14,,,,,, +SoftEther VPN Client,,,,t13i880900_fcb5b95cb75a_b0d3b4ac2a14,,,,,, +SoftEther VPN Server,,,,,,,d55f458d5a6c_d55f458d5a6c_0fc8c171b6ae,,, +Qakbot C2,,,,,,,2bab15409345_af684594efb4_000000000000,,, +Pikabot C2,,,,,,,1a59268f55e5_1a59268f55e5_795797892f9c,,,NOT Aegir +Darkgate Client,,,,,,po10nn060000_cdb958d032b0,,,, +Lumma Client,,,,,,po11nn050000_d253db9d024b,,,, +,,Epson Printer,,,,,,,28960_2-4-8-1-3_1460_3_1-4-8-16, +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12i190700_d83cc789557e_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over wininet +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12i210700_76e208dd3e22_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over winhttp +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12d190800_d83cc789557e_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over wininet +Cobalt Strike v4.9.1 beacon,,,Windows 10,t12d210800_76e208dd3e22_16bbda4055b2,t120300_c030_52d195ce1d92,,,,,Cobalt Strike v4.9.1 over winhttp +,,Ubiquiti Router,UniFi OS 3.2.12,,,,,,43440_2-4-8-1-3_1460_12_1-2-4-8-17, +,,Ubiquiti Camera,,,,,,,28960_2-4-8-1-3_1460_6_1-2-4-8-16, +,,Rachio IoT,,,,,,,7168_2-3-0_1460_0_0-0-0-0-0-0-0-0-0-0, +,,Phillips Hue Bridge,,,,,,,29200_2-1-1-4-1-3_1460_3_1-2-4-8-17, +,GoLang net package,,,t13d191000_9dc949149365_e7c285222651,,,,,, +,GoLang webhooks,,,t13d1412h2_e33ad33b3d25_6b314db333b6,,,,,, +,GoLang,,,t12d160700_8cdfa2d4673b_18dd7303c4a5,,,,,, +,GoLang,,,t13d141000_cbb2034c60b8_e7c285222651,,,,,, +Chromium Browser,,,,,,ge11nn08enus_050dd5cfb971_000000000000_000000000000,,,,Connection to new site over http 1.1 +Chromium Browser,,,,,,ge11nr07enus_a3e0c85f2f9f_000000000000_000000000000,,,,http 1.1 +Chromium Browser,,,,,,ge11nr08enus_307986a18ea3_000000000000_000000000000,,,,http 1.1 +ngrok 3.8.0,,,,,,ge11nn17enus_d488a5d46f42_000000000000_000000000000,,,,Chrome through ngrok +ngrok 3.8.0,,,,,,ge11nr15enus_a978addab902_000000000000_000000000000,,,,Chrome through ngrok +,,Cisco RV32x,,,,,fba10053814e_fba10053814e_795797892f9c,,5792_2-4-8-1-3_1460_4_3-6-12-24, +Curl,,,,,,ge11nn030000_fe444ad14866_000000000000_000000000000,,,, +Telegram,,,,,,po11nn07enus_b9cd978c9817_000000000000_000000000000,,,, +,,LIFX Smart Bulbs,,t12d520600_b380db6257eb_0a9c83bf8b96,,,,5744_2_1436_00,, +,,Nest,,t12d8008h1_9cedc1f1428b_046e095b7c4a,,,,29200_2-4-8-1-3_1460_6,, +,,Phillips Hue Bridge,,t12d350600_9d4c96c0953b_0a9c83bf8b96,,,,29200_2-1-1-4-1-3_1460_3,, +,WinINET,,Windows 10/11,t12d190800_d83cc789557e_7af1ed941c26,,,,,, +,GoLang,,,t12d190800_d83cc789557e_7af1ed941c26,,,,,, +,,Cloudflare,,,,,,,65535_2-4-8-1-3_1400_13_1, +,,Cloudflare,,,,,,65535_2-4-8-1-3_1400_13,,JA4TS when connecting to Cloudflare from Mac/Unix +,,Cloudflare,,,,,,65535_2-1-1-4-1-3_1400_13,,JA4TS when connecting to Cloudflare from Windows |